From 6b8806b45f970cb2446654d2c379f8dcaa93ce3c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 3 Aug 2024 12:34:11 -0700 Subject: [PATCH 1/9] feat(router.py): add flag for mock testing loadbalancing for rate limit errors --- litellm/proxy/_new_secret_config.yaml | 13 ++++++---- litellm/router.py | 34 ++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 238fe7136a6..47b93ccd2fa 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,7 +1,10 @@ model_list: - - model_name: "*" + - model_name: "gpt-4" litellm_params: - model: "*" - -# litellm_settings: -# failure_callback: ["langfuse"] + model: "gpt-4" + - model_name: "gpt-4" + litellm_params: + model: "gpt-4o" + - model_name: "gpt-4o-mini" + litellm_params: + model: "gpt-4o-mini" \ No newline at end of file diff --git a/litellm/router.py b/litellm/router.py index 108ca706c5f..0448139d2c8 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2468,6 +2468,8 @@ class Router: verbose_router_logger.info( f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" ) + if hasattr(original_exception, "message"): + original_exception.message += f"No fallback model group found for original model_group={model_group}. Fallbacks={fallbacks}" raise original_exception for mg in fallback_model_group: """ @@ -2492,14 +2494,19 @@ class Router: return response except Exception as e: raise e - except Exception as e: - verbose_router_logger.error(f"An exception occurred - {str(e)}") - verbose_router_logger.debug(traceback.format_exc()) + except Exception as new_exception: + verbose_router_logger.error( + "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}".format( + str(new_exception), traceback.format_exc() + ) + ) if hasattr(original_exception, "message"): # add the available fallbacks to the exception - original_exception.message += "\nReceived Model Group={}\nAvailable Model Group Fallbacks={}".format( - model_group, fallback_model_group + original_exception.message += "\nReceived Model Group={}\nAvailable Model Group Fallbacks={}\nCooldown Deployments={}".format( + model_group, + fallback_model_group, + await self._async_get_cooldown_deployments_with_debug_info(), ) raise original_exception @@ -2508,6 +2515,9 @@ class Router: f"Inside async function with retries: args - {args}; kwargs - {kwargs}" ) original_function = kwargs.pop("original_function") + mock_testing_rate_limit_error = kwargs.pop( + "mock_testing_rate_limit_error", None + ) fallbacks = kwargs.pop("fallbacks", self.fallbacks) context_window_fallbacks = kwargs.pop( "context_window_fallbacks", self.context_window_fallbacks @@ -2515,13 +2525,25 @@ class Router: content_policy_fallbacks = kwargs.pop( "content_policy_fallbacks", self.content_policy_fallbacks ) - + model_group = kwargs.get("model") num_retries = kwargs.pop("num_retries") verbose_router_logger.debug( f"async function w/ retries: original_function - {original_function}, num_retries - {num_retries}" ) try: + if ( + mock_testing_rate_limit_error is not None + and mock_testing_rate_limit_error is True + ): + verbose_router_logger.info( + "litellm.router.py::async_function_with_retries() - mock_testing_rate_limit_error=True. Raising litellm.RateLimitError." + ) + raise litellm.RateLimitError( + model=model_group, + llm_provider="", + message=f"This is a mock exception for model={model_group}, to trigger a rate limit error.", + ) # if the function call is successful, no exception will be raised and we'll break out of the loop response = await original_function(*args, **kwargs) return response From 1d892a41d21540e6f5a8bb7df900dc3bae16b805 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 3 Aug 2024 12:44:04 -0700 Subject: [PATCH 2/9] docs(proxy/reliability.md): add docs on testing if loadbalancing is working as expected --- docs/my-website/docs/proxy/reliability.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/reliability.md b/docs/my-website/docs/proxy/reliability.md index a3f03b3d76f..cb6550a4785 100644 --- a/docs/my-website/docs/proxy/reliability.md +++ b/docs/my-website/docs/proxy/reliability.md @@ -50,7 +50,7 @@ Detailed information about [routing strategies can be found here](../routing) $ litellm --config /path/to/config.yaml ``` -### Test - Load Balancing +### Test - Simple Call Here requests with model=gpt-3.5-turbo will be routed across multiple instances of azure/gpt-3.5-turbo @@ -138,6 +138,27 @@ print(response) +### Test - Loadbalancing + +In this request, the following will occur: +1. A rate limit exception will be raised +2. LiteLLM proxy will retry the request on the model group (default is 3). + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": "Hi there!"} + ], + "mock_testing_rate_limit_error": true +}' +``` + +[**See Code**](https://github.com/BerriAI/litellm/blob/6b8806b45f970cb2446654d2c379f8dcaa93ce3c/litellm/router.py#L2535) + ### Test - Client Side Fallbacks In this request the following will occur: 1. The request to `model="zephyr-beta"` will fail From 7a0792c918615142af0811cdfeb92fa445efe2ff Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 3 Aug 2024 12:49:39 -0700 Subject: [PATCH 3/9] fix(router.py): move deployment cooldown list message to error log, not client-side don't show user all deployments --- litellm/router.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 0448139d2c8..e31de5332eb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2496,17 +2496,18 @@ class Router: raise e except Exception as new_exception: verbose_router_logger.error( - "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}".format( - str(new_exception), traceback.format_exc() + "litellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}\n{}\n\nDebug Information:\nCooldown Deployments={}".format( + str(new_exception), + traceback.format_exc(), + await self._async_get_cooldown_deployments_with_debug_info(), ) ) if hasattr(original_exception, "message"): # add the available fallbacks to the exception - original_exception.message += "\nReceived Model Group={}\nAvailable Model Group Fallbacks={}\nCooldown Deployments={}".format( + original_exception.message += "\nReceived Model Group={}\nAvailable Model Group Fallbacks={}".format( model_group, fallback_model_group, - await self._async_get_cooldown_deployments_with_debug_info(), ) raise original_exception From 4a43f9f4110bf9ddbd10c7fada5da840c7386c1b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:24:23 -0700 Subject: [PATCH 4/9] docs supported models / providers --- docs/my-website/sidebars.js | 88 ++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 69fd32cb339..3e39348b971 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -83,50 +83,7 @@ const sidebars = { }, { type: "category", - label: "Completion()", - link: { - type: "generated-index", - title: "Completion()", - description: "Details on the completion() function", - slug: "/completion", - }, - items: [ - "completion/input", - "completion/provider_specific_params", - "completion/json_mode", - "completion/drop_params", - "completion/prompt_formatting", - "completion/output", - "exception_mapping", - "completion/stream", - "completion/message_trimming", - "completion/function_call", - "completion/vision", - "completion/model_alias", - "completion/batching", - "completion/mock_requests", - "completion/reliable_completions", - ], - }, - { - type: "category", - label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches(), Fine-Tuning()", - items: [ - "embedding/supported_embedding", - "embedding/async_embedding", - "embedding/moderation", - "image_generation", - "audio_transcription", - "text_to_speech", - "assistants", - "batches", - "fine_tuning", - "anthropic_completion" - ], - }, - { - type: "category", - label: "Supported Models & Providers", + label: "💯 Supported Models & Providers", link: { type: "generated-index", title: "Providers", @@ -183,6 +140,49 @@ const sidebars = { ], }, + { + type: "category", + label: "litellm.completion()", + link: { + type: "generated-index", + title: "Completion()", + description: "Details on the completion() function", + slug: "/completion", + }, + items: [ + "completion/input", + "completion/provider_specific_params", + "completion/json_mode", + "completion/drop_params", + "completion/prompt_formatting", + "completion/output", + "exception_mapping", + "completion/stream", + "completion/message_trimming", + "completion/function_call", + "completion/vision", + "completion/model_alias", + "completion/batching", + "completion/mock_requests", + "completion/reliable_completions", + ], + }, + { + type: "category", + label: "Embedding(), Image Generation(), Assistants(), Moderation(), Audio Transcriptions(), TTS(), Batches(), Fine-Tuning()", + items: [ + "embedding/supported_embedding", + "embedding/async_embedding", + "embedding/moderation", + "image_generation", + "audio_transcription", + "text_to_speech", + "assistants", + "batches", + "fine_tuning", + "anthropic_completion" + ], + }, "proxy/custom_pricing", "routing", "scheduler", From 1894aefd058a9c460a28f12889fe197f58eb33b0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:34:22 -0700 Subject: [PATCH 5/9] docs clean up organization --- docs/my-website/sidebars.js | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3e39348b971..afb778373f1 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -184,10 +184,6 @@ const sidebars = { ], }, "proxy/custom_pricing", - "routing", - "scheduler", - "set_keys", - "budget_manager", { type: "category", label: "Secret Manager", @@ -196,6 +192,22 @@ const sidebars = { "oidc" ] }, + { + type: "category", + label: "🚅 LiteLLM Python SDK", + items: [ + "routing", + "scheduler", + "set_keys", + "budget_manager", + "caching/all_caches", + { + type: "category", + label: "LangChain, LlamaIndex, Instructor Integration", + items: ["langchain/langchain", "tutorials/instructor"], + }, + ], + }, "completion/token_usage", "load_test", { @@ -227,14 +239,12 @@ const sidebars = { `observability/telemetry`, ], }, - "caching/all_caches", { type: "category", label: "Tutorials", items: [ 'tutorials/azure_openai', 'tutorials/instructor', - 'tutorials/oobabooga', "tutorials/gradio_integration", "tutorials/huggingface_codellama", "tutorials/huggingface_tutorial", @@ -246,11 +256,6 @@ const sidebars = { "tutorials/model_fallbacks", ], }, - { - type: "category", - label: "LangChain, LlamaIndex, Instructor Integration", - items: ["langchain/langchain", "tutorials/instructor"], - }, { type: "category", label: "Extras", From 942e77dfa848e1b2fab69058d9a3522f4aa2921f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:47:22 -0700 Subject: [PATCH 6/9] organize docs --- docs/my-website/docs/proxy/custom_pricing.md | 69 ++------------------ docs/my-website/docs/sdk_custom_pricing.md | 65 ++++++++++++++++++ docs/my-website/sidebars.js | 5 +- 3 files changed, 72 insertions(+), 67 deletions(-) create mode 100644 docs/my-website/docs/sdk_custom_pricing.md diff --git a/docs/my-website/docs/proxy/custom_pricing.md b/docs/my-website/docs/proxy/custom_pricing.md index 0b747f1193b..51634021b73 100644 --- a/docs/my-website/docs/proxy/custom_pricing.md +++ b/docs/my-website/docs/proxy/custom_pricing.md @@ -1,6 +1,6 @@ import Image from '@theme/IdealImage'; -# Custom Pricing - Sagemaker, etc. +# Custom LLM Pricing - Sagemaker, Azure, etc Use this to register custom pricing for models. @@ -16,39 +16,9 @@ LiteLLM already has pricing for any model in our [model cost map](https://github ::: -## Quick Start +## Cost Per Second (e.g. Sagemaker) -Register custom pricing for sagemaker completion model. - -For cost per second pricing, you **just** need to register `input_cost_per_second`. - -```python -# !pip install boto3 -from litellm import completion, completion_cost - -os.environ["AWS_ACCESS_KEY_ID"] = "" -os.environ["AWS_SECRET_ACCESS_KEY"] = "" -os.environ["AWS_REGION_NAME"] = "" - - -def test_completion_sagemaker(): - try: - print("testing sagemaker") - response = completion( - model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4", - messages=[{"role": "user", "content": "Hey, how's it going?"}], - input_cost_per_second=0.000420, - ) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - print(cost) - except Exception as e: - raise Exception(f"Error occurred: {e}") - -``` - -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server **Step 1: Add pricing to config.yaml** ```yaml @@ -75,38 +45,7 @@ litellm /path/to/config.yaml ## Cost Per Token (e.g. Azure) - -```python -# !pip install boto3 -from litellm import completion, completion_cost - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - - -def test_completion_azure_model(): - try: - print("testing azure custom pricing") - # azure call - response = completion( - model = "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}] - input_cost_per_token=0.005, - output_cost_per_token=1, - ) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - print(cost) - except Exception as e: - raise Exception(f"Error occurred: {e}") - -test_completion_azure_model() -``` - -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server ```yaml model_list: diff --git a/docs/my-website/docs/sdk_custom_pricing.md b/docs/my-website/docs/sdk_custom_pricing.md new file mode 100644 index 00000000000..c8577115109 --- /dev/null +++ b/docs/my-website/docs/sdk_custom_pricing.md @@ -0,0 +1,65 @@ +# Custom Pricing - SageMaker, Azure, etc + +Register custom pricing for sagemaker completion model. + +For cost per second pricing, you **just** need to register `input_cost_per_second`. + +```python +# !pip install boto3 +from litellm import completion, completion_cost + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + + +def test_completion_sagemaker(): + try: + print("testing sagemaker") + response = completion( + model="sagemaker/berri-benchmarking-Llama-2-70b-chat-hf-4", + messages=[{"role": "user", "content": "Hey, how's it going?"}], + input_cost_per_second=0.000420, + ) + # Add any assertions here to check the response + print(response) + cost = completion_cost(completion_response=response) + print(cost) + except Exception as e: + raise Exception(f"Error occurred: {e}") + +``` + + +## Cost Per Token (e.g. Azure) + + +```python +# !pip install boto3 +from litellm import completion, completion_cost + +## set ENV variables +os.environ["AZURE_API_KEY"] = "" +os.environ["AZURE_API_BASE"] = "" +os.environ["AZURE_API_VERSION"] = "" + + +def test_completion_azure_model(): + try: + print("testing azure custom pricing") + # azure call + response = completion( + model = "azure/", + messages = [{ "content": "Hello, how are you?","role": "user"}] + input_cost_per_token=0.005, + output_cost_per_token=1, + ) + # Add any assertions here to check the response + print(response) + cost = completion_cost(completion_response=response) + print(cost) + except Exception as e: + raise Exception(f"Error occurred: {e}") + +test_completion_azure_model() +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index afb778373f1..6674d91ac79 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -42,6 +42,7 @@ const sidebars = { "proxy/configs", "proxy/reliability", "proxy/cost_tracking", + "proxy/custom_pricing", "proxy/self_serve", "proxy/virtual_keys", { @@ -183,7 +184,6 @@ const sidebars = { "anthropic_completion" ], }, - "proxy/custom_pricing", { type: "category", label: "Secret Manager", @@ -199,6 +199,8 @@ const sidebars = { "routing", "scheduler", "set_keys", + "completion/token_usage", + "sdk_custom_pricing", "budget_manager", "caching/all_caches", { @@ -208,7 +210,6 @@ const sidebars = { }, ], }, - "completion/token_usage", "load_test", { type: "category", From 203cc35abce2ab3357240bd12a781d8c2e9e383d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:49:35 -0700 Subject: [PATCH 7/9] docs - use consistent name for LiteLLM proxy server --- README.md | 4 +- cookbook/litellm_router/error_log.txt | 152 +++++++++--------- cookbook/litellm_router/request_log.txt | 4 +- .../test_questions/question3.txt | 2 +- docs/my-website/docs/budget_manager.md | 4 +- docs/my-website/docs/index.md | 4 +- docs/my-website/docs/proxy/deploy.md | 10 +- docs/my-website/docs/proxy_server.md | 2 +- docs/my-website/docs/routing.md | 4 +- docs/my-website/docs/secret.md | 4 +- docs/my-website/docs/simple_proxy_old_doc.md | 2 +- docs/my-website/sidebars.js | 4 +- docs/my-website/src/pages/index.md | 2 +- 13 files changed, 99 insertions(+), 99 deletions(-) diff --git a/README.md b/README.md index 306f07ec26f..2153ae948ed 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@

Call all LLM APIs using the OpenAI format [Bedrock, Huggingface, VertexAI, TogetherAI, Azure, OpenAI, Groq etc.]

-

OpenAI Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

+

LiteLLM Proxy Server | Hosted Proxy (Preview) | Enterprise Tier

PyPI Version @@ -35,7 +35,7 @@ LiteLLM manages: - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Set Budgets & Rate limits per project, api key, model [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Set Budgets & Rate limits per project, api key, model [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) [**Jump to OpenAI Proxy Docs**](https://github.com/BerriAI/litellm?tab=readme-ov-file#openai-proxy---docs)
[**Jump to Supported LLM Providers**](https://github.com/BerriAI/litellm?tab=readme-ov-file#supported-providers-docs) diff --git a/cookbook/litellm_router/error_log.txt b/cookbook/litellm_router/error_log.txt index 6853ef4659a..983b47cbbba 100644 --- a/cookbook/litellm_router/error_log.txt +++ b/cookbook/litellm_router/error_log.txt @@ -1,10 +1,10 @@ -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -21,13 +21,13 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -49,7 +49,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -61,7 +61,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -70,7 +70,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -79,7 +79,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -109,7 +109,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -128,7 +128,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -148,7 +148,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -162,7 +162,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -174,7 +174,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -184,7 +184,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -193,19 +193,19 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -214,7 +214,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -234,7 +234,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -244,7 +244,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -253,7 +253,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -267,31 +267,31 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -305,7 +305,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -330,7 +330,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -339,7 +339,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -360,7 +360,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -369,7 +369,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -378,7 +378,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -388,7 +388,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -409,7 +409,7 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -422,13 +422,13 @@ Exception: Expecting value: line 1 column 1 (char 0) Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -438,7 +438,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: Expecting value: line 1 column 1 (char 0) -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -462,7 +462,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -482,7 +482,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -492,7 +492,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -516,7 +516,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -529,7 +529,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -546,13 +546,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -580,13 +580,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -624,7 +624,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -638,13 +638,13 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -660,7 +660,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -681,7 +681,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -691,31 +691,31 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -771,7 +771,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -780,7 +780,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -800,7 +800,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -820,7 +820,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -830,7 +830,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -840,7 +840,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -850,7 +850,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -862,13 +862,13 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -877,7 +877,7 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -898,7 +898,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -919,7 +919,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -936,19 +936,19 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -961,25 +961,25 @@ Exception: 'Response' object has no attribute 'get' Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. Call all LLM APIs using the Ope Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -993,7 +993,7 @@ Question: Given this context, what is litellm? LiteLLM about: About Call all LLM APIs using the OpenAI format. Exception: 'Response' object has no attribute 'get' -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 diff --git a/cookbook/litellm_router/request_log.txt b/cookbook/litellm_router/request_log.txt index 0aed7490496..821d87ab56a 100644 --- a/cookbook/litellm_router/request_log.txt +++ b/cookbook/litellm_router/request_log.txt @@ -20,7 +20,7 @@ Call all LLM APIs using the OpenAI format. Response ID: 52dbbd49-eedb-4c11-8382-3ca7deb1af35 Url: /queue/response/52dbbd49-eedb-4c11-8382-3ca7deb1af35 Time: 3.50 seconds -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 @@ -35,7 +35,7 @@ Question: Does litellm support ooobagooba llms? how can i call oobagooba llms. C Response ID: ae1e2b71-d711-456d-8df0-13ce0709eb04 Url: /queue/response/ae1e2b71-d711-456d-8df0-13ce0709eb04 Time: 5.60 seconds -Question: What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +Question: What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 10 diff --git a/cookbook/litellm_router/test_questions/question3.txt b/cookbook/litellm_router/test_questions/question3.txt index a122787504e..d6006f9c73c 100644 --- a/cookbook/litellm_router/test_questions/question3.txt +++ b/cookbook/litellm_router/test_questions/question3.txt @@ -1,4 +1,4 @@ -What endpoints does the litellm proxy have 💥 OpenAI Proxy Server +What endpoints does the litellm proxy have 💥 LiteLLM Proxy Server LiteLLM Server manages: Calling 100+ LLMs Huggingface/Bedrock/TogetherAI/etc. in the OpenAI ChatCompletions & Completions format diff --git a/docs/my-website/docs/budget_manager.md b/docs/my-website/docs/budget_manager.md index 1a2c7e7eecd..6bea96ef9ce 100644 --- a/docs/my-website/docs/budget_manager.md +++ b/docs/my-website/docs/budget_manager.md @@ -7,14 +7,14 @@ Don't want to get crazy bills because either while you're calling LLM APIs **or* :::info -If you want a server to manage user keys, budgets, etc. use our [OpenAI Proxy Server](./proxy/virtual_keys.md) +If you want a server to manage user keys, budgets, etc. use our [LiteLLM Proxy Server](./proxy/virtual_keys.md) ::: LiteLLM exposes: * `litellm.max_budget`: a global variable you can use to set the max budget (in USD) across all your litellm calls. If this budget is exceeded, it will raise a BudgetExceededError * `BudgetManager`: A class to help set budgets per user. BudgetManager creates a dictionary to manage the user budgets, where the key is user and the object is their current cost + model-specific costs. -* `OpenAI Proxy Server`: A server to call 100+ LLMs with an openai-compatible endpoint. Manages user budgets, spend tracking, load balancing etc. +* `LiteLLM Proxy Server`: A server to call 100+ LLMs with an openai-compatible endpoint. Manages user budgets, spend tracking, load balancing etc. ## quick start diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index 6b472ee6c6d..a560ecf76da 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -10,11 +10,11 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## How to use LiteLLM You can use litellm through either: -1. [OpenAI proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects +1. [LiteLLM Proxy Server](#openai-proxy) - Server to call 100+ LLMs, load balance, cost tracking across projects 2. [LiteLLM python SDK](#basic-usage) - Python Client to call 100+ LLMs, load balance, cost tracking ## LiteLLM Python SDK diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 35fc0a50866..c7617196e58 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -246,7 +246,7 @@ helm install lite-helm ./litellm-helm kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. @@ -301,7 +301,7 @@ docker run \ --config /app/config.yaml --detailed_debug ``` -Your OpenAI proxy server is now running on `http://0.0.0.0:4000`. +Your LiteLLM Proxy Server is now running on `http://0.0.0.0:4000`. @@ -399,7 +399,7 @@ kubectl apply -f /path/to/service.yaml kubectl port-forward service/litellm-service 4000:4000 ``` -Your OpenAI proxy server is now running on `http://0.0.0.0:4000`. +Your LiteLLM Proxy Server is now running on `http://0.0.0.0:4000`. @@ -441,7 +441,7 @@ kubectl \ 4000:4000 ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. If you need to set your litellm proxy config.yaml, you can find this in [values.yaml](https://github.com/BerriAI/litellm/blob/main/deploy/charts/litellm-helm/values.yaml) @@ -486,7 +486,7 @@ helm install lite-helm ./litellm-helm kubectl --namespace default port-forward $POD_NAME 8080:$CONTAINER_PORT ``` -Your OpenAI proxy server is now running on `http://127.0.0.1:4000`. +Your LiteLLM Proxy Server is now running on `http://127.0.0.1:4000`. diff --git a/docs/my-website/docs/proxy_server.md b/docs/my-website/docs/proxy_server.md index ef9352ab1f7..0d08db7444e 100644 --- a/docs/my-website/docs/proxy_server.md +++ b/docs/my-website/docs/proxy_server.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# [OLD PROXY 👉 [NEW proxy here](./simple_proxy)] Local OpenAI Proxy Server +# [OLD PROXY 👉 [NEW proxy here](./simple_proxy)] Local LiteLLM Proxy Server A fast, and lightweight OpenAI-compatible server to call 100+ LLM APIs. diff --git a/docs/my-website/docs/routing.md b/docs/my-website/docs/routing.md index 905954e9797..d83755e68dd 100644 --- a/docs/my-website/docs/routing.md +++ b/docs/my-website/docs/routing.md @@ -14,7 +14,7 @@ In production, litellm supports using Redis as a way to track cooldown server an :::info -If you want a server to load balance across different LLM APIs, use our [OpenAI Proxy Server](./proxy/load_balancing.md) +If you want a server to load balance across different LLM APIs, use our [LiteLLM Proxy Server](./proxy/load_balancing.md) ::: @@ -1637,7 +1637,7 @@ response = router.completion( ## Deploy Router -If you want a server to load balance across different LLM APIs, use our [OpenAI Proxy Server](./simple_proxy#load-balancing---multiple-instances-of-1-model) +If you want a server to load balance across different LLM APIs, use our [LiteLLM Proxy Server](./simple_proxy#load-balancing---multiple-instances-of-1-model) ## Init Params for the litellm.Router diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index 91ae383686e..c44f2cd10cb 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -90,7 +90,7 @@ litellm.secret_manager = client litellm.get_secret("your-test-key") ``` -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server 1. Install Proxy dependencies ```bash @@ -129,7 +129,7 @@ litellm --config /path/to/config.yaml Use encrypted keys from Google KMS on the proxy -### Usage with OpenAI Proxy Server +### Usage with LiteLLM Proxy Server ## Step 1. Add keys to env ``` diff --git a/docs/my-website/docs/simple_proxy_old_doc.md b/docs/my-website/docs/simple_proxy_old_doc.md index 195728d1be4..2d68db32964 100644 --- a/docs/my-website/docs/simple_proxy_old_doc.md +++ b/docs/my-website/docs/simple_proxy_old_doc.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 💥 OpenAI Proxy Server +# 💥 LiteLLM Proxy Server LiteLLM Server manages: diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 6674d91ac79..e57f340c70c 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -20,10 +20,10 @@ const sidebars = { { type: "doc", id: "index" }, // NEW { type: "category", - label: "💥 OpenAI Proxy Server", + label: "💥 LiteLLM Proxy Server", link: { type: "generated-index", - title: "💥 OpenAI Proxy Server", + title: "💥 LiteLLM Proxy Server", description: `Proxy Server to call 100+ LLMs in a unified interface & track spend, set budgets per virtual key/user`, slug: "/simple_proxy", }, diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index 308ed083175..36d47aedf7c 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -10,7 +10,7 @@ https://github.com/BerriAI/litellm - Translate inputs to provider's `completion`, `embedding`, and `image_generation` endpoints - [Consistent output](https://docs.litellm.ai/docs/completion/output), text responses will always be available at `['choices'][0]['message']['content']` - Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [OpenAI Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) ## Basic usage From 58de3f948650a57b2595ab211e67592630afad30 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 3 Aug 2024 12:53:35 -0700 Subject: [PATCH 8/9] fix(vertex_httpx.py): fix linting error --- litellm/llms/vertex_httpx.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/litellm/llms/vertex_httpx.py b/litellm/llms/vertex_httpx.py index 9995373f32e..954a30b8010 100644 --- a/litellm/llms/vertex_httpx.py +++ b/litellm/llms/vertex_httpx.py @@ -13,6 +13,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union import httpx # type: ignore import requests # type: ignore +from openai.types.image import Image import litellm import litellm.litellm_core_utils @@ -1341,10 +1342,10 @@ class VertexLLM(BaseLLM): _json_response = response.json() _predictions = _json_response["predictions"] - _response_data: List[litellm.ImageObject] = [] + _response_data: List[Image] = [] for _prediction in _predictions: _bytes_base64_encoded = _prediction["bytesBase64Encoded"] - image_object = litellm.ImageObject(b64_json=_bytes_base64_encoded) + image_object = Image(b64_json=_bytes_base64_encoded) _response_data.append(image_object) model_response.data = _response_data @@ -1453,10 +1454,10 @@ class VertexLLM(BaseLLM): _json_response = response.json() _predictions = _json_response["predictions"] - _response_data: List[litellm.ImageObject] = [] + _response_data: List[Image] = [] for _prediction in _predictions: _bytes_base64_encoded = _prediction["bytesBase64Encoded"] - image_object = litellm.ImageObject(b64_json=_bytes_base64_encoded) + image_object = Image(b64_json=_bytes_base64_encoded) _response_data.append(image_object) model_response.data = _response_data From cfdbb3d2374ddc576ed7e3c4f8e8041efee84dca Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 3 Aug 2024 12:53:47 -0700 Subject: [PATCH 9/9] docs secret manager --- docs/my-website/docs/secret.md | 31 ++++--------------------------- docs/my-website/sidebars.js | 16 ++++++++-------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/docs/my-website/docs/secret.md b/docs/my-website/docs/secret.md index c44f2cd10cb..c2b6774c0b1 100644 --- a/docs/my-website/docs/secret.md +++ b/docs/my-website/docs/secret.md @@ -61,7 +61,7 @@ litellm --config /path/to/config.yaml ``` ## Azure Key Vault - + ### Usage with LiteLLM Proxy Server @@ -160,29 +160,6 @@ $ litellm --test [Quick Test Proxy](./proxy/quick_start#using-litellm-proxy---curl-request-openai-package-langchain-langchain-js) - -## Infisical Secret Manager -Integrates with [Infisical's Secret Manager](https://infisical.com/) for secure storage and retrieval of API keys and sensitive data. - -### Usage -liteLLM manages reading in your LLM API secrets/env variables from Infisical for you - -```python -import litellm -from infisical import InfisicalClient - -litellm.secret_manager = InfisicalClient(token="your-token") - -messages = [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "What's the weather like today?"}, -] - -response = litellm.completion(model="gpt-3.5-turbo", messages=messages) - -print(response) -``` - - + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index e57f340c70c..27084f3b451 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -50,6 +50,14 @@ const sidebars = { label: "🪢 Logging", items: ["proxy/logging", "proxy/bucket", "proxy/streaming_logging"], }, + { + type: "category", + label: "Secret Manager - storing LLM API Keys", + items: [ + "secret", + "oidc" + ] + }, "proxy/team_logging", "proxy/guardrails", "proxy/tag_routing", @@ -184,14 +192,6 @@ const sidebars = { "anthropic_completion" ], }, - { - type: "category", - label: "Secret Manager", - items: [ - "secret", - "oidc" - ] - }, { type: "category", label: "🚅 LiteLLM Python SDK",