From 318b4813f2b45ecf98bd46104d560393768b832c Mon Sep 17 00:00:00 2001 From: mogith-pn <143642606+mogith-pn@users.noreply.github.com> Date: Tue, 30 Apr 2024 22:38:33 +0530 Subject: [PATCH 001/169] Clarifai-LiteLLM integration (#1) * intg v1 clarifai-litellm * Added more community models and testcase * Clarifai-updated markdown docs --- cookbook/liteLLM_clarifai_Demo.ipynb | 151 ++++++++++++++ docs/my-website/docs/providers/clarifai.md | 177 +++++++++++++++++ litellm/__init__.py | 70 +++++++ litellm/llms/clarifai.py | 216 +++++++++++++++++++++ litellm/llms/prompt_templates/factory.py | 3 + litellm/main.py | 50 +++++ litellm/tests/test_clarifai_completion.py | 67 +++++++ 7 files changed, 734 insertions(+) create mode 100644 cookbook/liteLLM_clarifai_Demo.ipynb create mode 100644 docs/my-website/docs/providers/clarifai.md create mode 100644 litellm/llms/clarifai.py create mode 100644 litellm/tests/test_clarifai_completion.py diff --git a/cookbook/liteLLM_clarifai_Demo.ipynb b/cookbook/liteLLM_clarifai_Demo.ipynb new file mode 100644 index 00000000000..4e3b4dbb06a --- /dev/null +++ b/cookbook/liteLLM_clarifai_Demo.ipynb @@ -0,0 +1,151 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# LiteLLM Clarifai \n", + "This notebook walks you through on how to use liteLLM integration of Clarifai and call LLM model from clarifai with response in openAI output format." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pre-Requisites" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#install necessary packages\n", + "!pip install litellm\n", + "!pip install clarifai" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To obtain Clarifai Personal Access Token follow the steps mentioned in the [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/)" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "## Set Clarifai Credentials\n", + "import os\n", + "os.environ[\"CLARIFAI_API_KEY\"]= \"YOUR_CLARIFAI_PAT\" # Clarifai PAT" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Mistral-large" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import litellm\n", + "\n", + "litellm.set_verbose=False" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Mistral large response : ModelResponse(id='chatcmpl-6eed494d-7ae2-4870-b9c2-6a64d50a6151', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\"In the grand tapestry of time, where tales unfold,\\nLies the chronicle of ages, a sight to behold.\\nA tale of empires rising, and kings of old,\\nOf civilizations lost, and stories untold.\\n\\nOnce upon a yesterday, in a time so vast,\\nHumans took their first steps, casting shadows in the past.\\nFrom the cradle of mankind, a journey they embarked,\\nThrough stone and bronze and iron, their skills they sharpened and marked.\\n\\nEgyptians built pyramids, reaching for the skies,\\nWhile Greeks sought wisdom, truth, in philosophies that lie.\\nRoman legions marched, their empire to expand,\\nAnd in the East, the Silk Road joined the world, hand in hand.\\n\\nThe Middle Ages came, with knights in shining armor,\\nFeudal lords and serfs, a time of both clamor and calm order.\\nThen Renaissance bloomed, like a flower in the sun,\\nA rebirth of art and science, a new age had begun.\\n\\nAcross the vast oceans, explorers sailed with courage bold,\\nDiscovering new lands, stories of adventure, untold.\\nIndustrial Revolution churned, progress in its wake,\\nMachines and factories, a whole new world to make.\\n\\nTwo World Wars raged, a testament to man's strife,\\nYet from the ashes rose hope, a renewed will for life.\\nInto the modern era, technology took flight,\\nConnecting every corner, bathed in digital light.\\n\\nHistory, a symphony, a melody of time,\\nA testament to human will, resilience so sublime.\\nIn every page, a lesson, in every tale, a guide,\\nFor understanding our past, shapes our future's tide.\", role='assistant'))], created=1713896412, model='https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=13, completion_tokens=338, total_tokens=351))\n" + ] + } + ], + "source": [ + "from litellm import completion\n", + "\n", + "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", + "response=completion(\n", + " model=\"clarifai/mistralai.completion.mistral-large\",\n", + " messages=messages,\n", + " )\n", + "\n", + "print(f\"Mistral large response : {response}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Claude-2.1 " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Claude-2.1 response : ModelResponse(id='chatcmpl-d126c919-4db4-4aa3-ac8f-7edea41e0b93', choices=[Choices(finish_reason='stop', index=1, message=Message(content=\" Here's a poem I wrote about history:\\n\\nThe Tides of Time\\n\\nThe tides of time ebb and flow,\\nCarrying stories of long ago.\\nFigures and events come into light,\\nShaping the future with all their might.\\n\\nKingdoms rise, empires fall, \\nLeaving traces that echo down every hall.\\nRevolutions bring change with a fiery glow,\\nToppling structures from long ago.\\n\\nExplorers traverse each ocean and land,\\nSeeking treasures they don't understand.\\nWhile artists and writers try to make their mark,\\nHoping their works shine bright in the dark.\\n\\nThe cycle repeats again and again,\\nAs humanity struggles to learn from its pain.\\nThough the players may change on history's stage,\\nThe themes stay the same from age to age.\\n\\nWar and peace, life and death,\\nLove and strife with every breath.\\nThe tides of time continue their dance,\\nAs we join in, by luck or by chance.\\n\\nSo we study the past to light the way forward, \\nHeeding warnings from stories told and heard.\\nThe future unfolds from this unending flow -\\nWhere the tides of time ultimately go.\", role='assistant'))], created=1713896579, model='https://api.clarifai.com/v2/users/anthropic/apps/completion/models/claude-2_1/outputs', object='chat.completion', system_fingerprint=None, usage=Usage(prompt_tokens=12, completion_tokens=232, total_tokens=244))\n" + ] + } + ], + "source": [ + "from litellm import completion\n", + "\n", + "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", + "response=completion(\n", + " model=\"clarifai/anthropic.completion.claude-2_1\",\n", + " messages=messages,\n", + " )\n", + "\n", + "print(f\"Claude-2.1 response : {response}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/my-website/docs/providers/clarifai.md b/docs/my-website/docs/providers/clarifai.md new file mode 100644 index 00000000000..acc8c54befe --- /dev/null +++ b/docs/my-website/docs/providers/clarifai.md @@ -0,0 +1,177 @@ + +# Clarifai +Anthropic, OpenAI, Mistral, Llama and Gemini LLMs are Supported on Clarifai. + +## Pre-Requisites + +`pip install clarifai` + +`pip install litellm` + +## Required Environment Variables +To obtain your Clarifai Personal access token follow this [link](https://docs.clarifai.com/clarifai-basics/authentication/personal-access-tokens/). Optionally the PAT can also be passed in `completion` function. + +```python +os.environ["CALRIFAI_API_KEY"] = "YOUR_CLARIFAI_PAT" # CLARIFAI_PAT +``` + +## Usage + +```python +import os +from litellm import completion + +os.environ["CLARIFAI_API_KEY"] = "" + +response = completion( + model="clarifai/mistralai.completion.mistral-large", + messages=[{ "content": "Tell me a joke about physics?","role": "user"}] +) +``` + +**Output** +```json +{ + "id": "chatcmpl-572701ee-9ab2-411c-ac75-46c1ba18e781", + "choices": [ + { + "finish_reason": "stop", + "index": 1, + "message": { + "content": "Sure, here's a physics joke for you:\n\nWhy can't you trust an atom?\n\nBecause they make up everything!", + "role": "assistant" + } + } + ], + "created": 1714410197, + "model": "https://api.clarifai.com/v2/users/mistralai/apps/completion/models/mistral-large/outputs", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "prompt_tokens": 14, + "completion_tokens": 24, + "total_tokens": 38 + } + } +``` + +## Clarifai models +liteLLM supports non-streaming requests to all models on [Clarifai community](https://clarifai.com/explore/models?filterData=%5B%7B%22field%22%3A%22use_cases%22%2C%22value%22%3A%5B%22llm%22%5D%7D%5D&page=1&perPage=24) + +Example Usage - Note: liteLLM supports all models deployed on Clarifai + +## Llama LLMs +| Model Name | Function Call | +---------------------------|---------------------------------| +| clarifai/meta.Llama-2.llama2-7b-chat | `completion('clarifai/meta.Llama-2.llama2-7b-chat', messages)` +| clarifai/meta.Llama-2.llama2-13b-chat | `completion('clarifai/meta.Llama-2.llama2-13b-chat', messages)` +| clarifai/meta.Llama-2.llama2-70b-chat | `completion('clarifai/meta.Llama-2.llama2-70b-chat', messages)` | +| clarifai/meta.Llama-2.codeLlama-70b-Python | `completion('clarifai/meta.Llama-2.codeLlama-70b-Python', messages)`| +| clarifai/meta.Llama-2.codeLlama-70b-Instruct | `completion('clarifai/meta.Llama-2.codeLlama-70b-Instruct', messages)` | + +## Mistal LLMs +| Model Name | Function Call | +|---------------------------------------------|------------------------------------------------------------------------| +| clarifai/mistralai.completion.mixtral-8x22B | `completion('clarifai/mistralai.completion.mixtral-8x22B', messages)` | +| clarifai/mistralai.completion.mistral-large | `completion('clarifai/mistralai.completion.mistral-large', messages)` | +| clarifai/mistralai.completion.mistral-medium | `completion('clarifai/mistralai.completion.mistral-medium', messages)` | +| clarifai/mistralai.completion.mistral-small | `completion('clarifai/mistralai.completion.mistral-small', messages)` | +| clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1 | `completion('clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1', messages)` +| clarifai/mistralai.completion.mistral-7B-OpenOrca | `completion('clarifai/mistralai.completion.mistral-7B-OpenOrca', messages)` | +| clarifai/mistralai.completion.openHermes-2-mistral-7B | `completion('clarifai/mistralai.completion.openHermes-2-mistral-7B', messages)` | + + +## Jurassic LLMs +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/ai21.complete.Jurassic2-Grande | `completion('clarifai/ai21.complete.Jurassic2-Grande', messages)` | +| clarifai/ai21.complete.Jurassic2-Grande-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Grande-Instruct', messages)` | +| clarifai/ai21.complete.Jurassic2-Jumbo-Instruct | `completion('clarifai/ai21.complete.Jurassic2-Jumbo-Instruct', messages)` | +| clarifai/ai21.complete.Jurassic2-Jumbo | `completion('clarifai/ai21.complete.Jurassic2-Jumbo', messages)` | +| clarifai/ai21.complete.Jurassic2-Large | `completion('clarifai/ai21.complete.Jurassic2-Large', messages)` | + +## Wizard LLMs + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/wizardlm.generate.wizardCoder-Python-34B | `completion('clarifai/wizardlm.generate.wizardCoder-Python-34B', messages)` | +| clarifai/wizardlm.generate.wizardLM-70B | `completion('clarifai/wizardlm.generate.wizardLM-70B', messages)` | +| clarifai/wizardlm.generate.wizardLM-13B | `completion('clarifai/wizardlm.generate.wizardLM-13B', messages)` | +| clarifai/wizardlm.generate.wizardCoder-15B | `completion('clarifai/wizardlm.generate.wizardCoder-15B', messages)` | + +## Anthropic models + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/anthropic.completion.claude-v1 | `completion('clarifai/anthropic.completion.claude-v1', messages)` | +| clarifai/anthropic.completion.claude-instant-1_2 | `completion('clarifai/anthropic.completion.claude-instant-1_2', messages)` | +| clarifai/anthropic.completion.claude-instant | `completion('clarifai/anthropic.completion.claude-instant', messages)` | +| clarifai/anthropic.completion.claude-v2 | `completion('clarifai/anthropic.completion.claude-v2', messages)` | +| clarifai/anthropic.completion.claude-2_1 | `completion('clarifai/anthropic.completion.claude-2_1', messages)` | +| clarifai/anthropic.completion.claude-3-opus | `completion('clarifai/anthropic.completion.claude-3-opus', messages)` | +| clarifai/anthropic.completion.claude-3-sonnet | `completion('clarifai/anthropic.completion.claude-3-sonnet', messages)` | + +## OpenAI GPT LLMs + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/openai.chat-completion.GPT-4 | `completion('clarifai/openai.chat-completion.GPT-4', messages)` | +| clarifai/openai.chat-completion.GPT-3_5-turbo | `completion('clarifai/openai.chat-completion.GPT-3_5-turbo', messages)` | +| clarifai/openai.chat-completion.gpt-4-turbo | `completion('clarifai/openai.chat-completion.gpt-4-turbo', messages)` | +| clarifai/openai.completion.gpt-3_5-turbo-instruct | `completion('clarifai/openai.completion.gpt-3_5-turbo-instruct', messages)` | + +## GCP LLMs + +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/gcp.generate.gemini-1_5-pro | `completion('clarifai/gcp.generate.gemini-1_5-pro', messages)` | +| clarifai/gcp.generate.imagen-2 | `completion('clarifai/gcp.generate.imagen-2', messages)` | +| clarifai/gcp.generate.code-gecko | `completion('clarifai/gcp.generate.code-gecko', messages)` | +| clarifai/gcp.generate.code-bison | `completion('clarifai/gcp.generate.code-bison', messages)` | +| clarifai/gcp.generate.text-bison | `completion('clarifai/gcp.generate.text-bison', messages)` | +| clarifai/gcp.generate.gemma-2b-it | `completion('clarifai/gcp.generate.gemma-2b-it', messages)` | +| clarifai/gcp.generate.gemma-7b-it | `completion('clarifai/gcp.generate.gemma-7b-it', messages)` | +| clarifai/gcp.generate.gemini-pro | `completion('clarifai/gcp.generate.gemini-pro', messages)` | +| clarifai/gcp.generate.gemma-1_1-7b-it | `completion('clarifai/gcp.generate.gemma-1_1-7b-it', messages)` | + +## Cohere LLMs +| Model Name | Function Call | +|-----------------------------------------------|---------------------------------------------------------------------| +| clarifai/cohere.generate.cohere-generate-command | `completion('clarifai/cohere.generate.cohere-generate-command', messages)` | + clarifai/cohere.generate.command-r-plus' | `completion('clarifai/clarifai/cohere.generate.command-r-plus', messages)`| + +## Databricks LLMs + +| Model Name | Function Call | +|---------------------------------------------------|---------------------------------------------------------------------| +| clarifai/databricks.drbx.dbrx-instruct | `completion('clarifai/databricks.drbx.dbrx-instruct', messages)` | +| clarifai/databricks.Dolly-v2.dolly-v2-12b | `completion('clarifai/databricks.Dolly-v2.dolly-v2-12b', messages)`| + +## Microsoft LLMs + +| Model Name | Function Call | +|---------------------------------------------------|---------------------------------------------------------------------| +| clarifai/microsoft.text-generation.phi-2 | `completion('clarifai/microsoft.text-generation.phi-2', messages)` | +| clarifai/microsoft.text-generation.phi-1_5 | `completion('clarifai/microsoft.text-generation.phi-1_5', messages)`| + +## Salesforce models + +| Model Name | Function Call | +|-----------------------------------------------------------|-------------------------------------------------------------------------------| +| clarifai/salesforce.blip.general-english-image-caption-blip-2 | `completion('clarifai/salesforce.blip.general-english-image-caption-blip-2', messages)` | +| clarifai/salesforce.xgen.xgen-7b-8k-instruct | `completion('clarifai/salesforce.xgen.xgen-7b-8k-instruct', messages)` | + + +## Other Top performing LLMs + +| Model Name | Function Call | +|---------------------------------------------------|---------------------------------------------------------------------| +| clarifai/deci.decilm.deciLM-7B-instruct | `completion('clarifai/deci.decilm.deciLM-7B-instruct', messages)` | +| clarifai/upstage.solar.solar-10_7b-instruct | `completion('clarifai/upstage.solar.solar-10_7b-instruct', messages)` | +| clarifai/openchat.openchat.openchat-3_5-1210 | `completion('clarifai/openchat.openchat.openchat-3_5-1210', messages)` | +| clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B | `completion('clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B', messages)` | +| clarifai/fblgit.una-cybertron.una-cybertron-7b-v2 | `completion('clarifai/fblgit.una-cybertron.una-cybertron-7b-v2', messages)` | +| clarifai/tiiuae.falcon.falcon-40b-instruct | `completion('clarifai/tiiuae.falcon.falcon-40b-instruct', messages)` | +| clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat | `completion('clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat', messages)` | +| clarifai/bigcode.code.StarCoder | `completion('clarifai/bigcode.code.StarCoder', messages)` | +| clarifai/mosaicml.mpt.mpt-7b-instruct | `completion('clarifai/mosaicml.mpt.mpt-7b-instruct', messages)` | diff --git a/litellm/__init__.py b/litellm/__init__.py index 5ef78dce4d5..8523ee577ed 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -49,6 +49,7 @@ azure_key: Optional[str] = None anthropic_key: Optional[str] = None replicate_key: Optional[str] = None cohere_key: Optional[str] = None +clarifai_key: Optional[str] = None maritalk_key: Optional[str] = None ai21_key: Optional[str] = None openrouter_key: Optional[str] = None @@ -366,6 +367,73 @@ replicate_models: List = [ "replit/replit-code-v1-3b:b84f4c074b807211cd75e3e8b1589b6399052125b4c27106e43d47189e8415ad", ] +clarifai_models: List = [ + 'clarifai/meta.Llama-3.Llama-3-8B-Instruct', + 'clarifai/gcp.generate.gemma-1_1-7b-it', + 'clarifai/mistralai.completion.mixtral-8x22B', + 'clarifai/cohere.generate.command-r-plus', + 'clarifai/databricks.drbx.dbrx-instruct', + 'clarifai/mistralai.completion.mistral-large', + 'clarifai/mistralai.completion.mistral-medium', + 'clarifai/mistralai.completion.mistral-small', + 'clarifai/mistralai.completion.mixtral-8x7B-Instruct-v0_1', + 'clarifai/gcp.generate.gemma-2b-it', + 'clarifai/gcp.generate.gemma-7b-it', + 'clarifai/deci.decilm.deciLM-7B-instruct', + 'clarifai/mistralai.completion.mistral-7B-Instruct', + 'clarifai/gcp.generate.gemini-pro', + 'clarifai/anthropic.completion.claude-v1', + 'clarifai/anthropic.completion.claude-instant-1_2', + 'clarifai/anthropic.completion.claude-instant', + 'clarifai/anthropic.completion.claude-v2', + 'clarifai/anthropic.completion.claude-2_1', + 'clarifai/meta.Llama-2.codeLlama-70b-Python', + 'clarifai/meta.Llama-2.codeLlama-70b-Instruct', + 'clarifai/openai.completion.gpt-3_5-turbo-instruct', + 'clarifai/meta.Llama-2.llama2-7b-chat', + 'clarifai/meta.Llama-2.llama2-13b-chat', + 'clarifai/meta.Llama-2.llama2-70b-chat', + 'clarifai/openai.chat-completion.gpt-4-turbo', + 'clarifai/microsoft.text-generation.phi-2', + 'clarifai/meta.Llama-2.llama2-7b-chat-vllm', + 'clarifai/upstage.solar.solar-10_7b-instruct', + 'clarifai/openchat.openchat.openchat-3_5-1210', + 'clarifai/togethercomputer.stripedHyena.stripedHyena-Nous-7B', + 'clarifai/gcp.generate.text-bison', + 'clarifai/meta.Llama-2.llamaGuard-7b', + 'clarifai/fblgit.una-cybertron.una-cybertron-7b-v2', + 'clarifai/openai.chat-completion.GPT-4', + 'clarifai/openai.chat-completion.GPT-3_5-turbo', + 'clarifai/ai21.complete.Jurassic2-Grande', + 'clarifai/ai21.complete.Jurassic2-Grande-Instruct', + 'clarifai/ai21.complete.Jurassic2-Jumbo-Instruct', + 'clarifai/ai21.complete.Jurassic2-Jumbo', + 'clarifai/ai21.complete.Jurassic2-Large', + 'clarifai/cohere.generate.cohere-generate-command', + 'clarifai/wizardlm.generate.wizardCoder-Python-34B', + 'clarifai/wizardlm.generate.wizardLM-70B', + 'clarifai/tiiuae.falcon.falcon-40b-instruct', + 'clarifai/togethercomputer.RedPajama.RedPajama-INCITE-7B-Chat', + 'clarifai/gcp.generate.code-gecko', + 'clarifai/gcp.generate.code-bison', + 'clarifai/mistralai.completion.mistral-7B-OpenOrca', + 'clarifai/mistralai.completion.openHermes-2-mistral-7B', + 'clarifai/wizardlm.generate.wizardLM-13B', + 'clarifai/huggingface-research.zephyr.zephyr-7B-alpha', + 'clarifai/wizardlm.generate.wizardCoder-15B', + 'clarifai/microsoft.text-generation.phi-1_5', + 'clarifai/databricks.Dolly-v2.dolly-v2-12b', + 'clarifai/bigcode.code.StarCoder', + 'clarifai/salesforce.xgen.xgen-7b-8k-instruct', + 'clarifai/mosaicml.mpt.mpt-7b-instruct', + 'clarifai/anthropic.completion.claude-3-opus', + 'clarifai/anthropic.completion.claude-3-sonnet', + 'clarifai/gcp.generate.gemini-1_5-pro', + 'clarifai/gcp.generate.imagen-2', + 'clarifai/salesforce.blip.general-english-image-caption-blip-2', +] + + huggingface_models: List = [ "meta-llama/Llama-2-7b-hf", "meta-llama/Llama-2-7b-chat-hf", @@ -470,6 +538,7 @@ provider_list: List = [ "text-completion-openai", "cohere", "cohere_chat", + "clarifai", "anthropic", "replicate", "huggingface", @@ -608,6 +677,7 @@ from .llms.anthropic import AnthropicConfig from .llms.anthropic_text import AnthropicTextConfig from .llms.replicate import ReplicateConfig from .llms.cohere import CohereConfig +from .llms.clarifai import ClarifaiConfig from .llms.ai21 import AI21Config from .llms.together_ai import TogetherAIConfig from .llms.cloudflare import CloudflareConfig diff --git a/litellm/llms/clarifai.py b/litellm/llms/clarifai.py new file mode 100644 index 00000000000..2a7d77c61c6 --- /dev/null +++ b/litellm/llms/clarifai.py @@ -0,0 +1,216 @@ +import os, types, traceback +import json +import requests +import time +from typing import Callable, Optional +from litellm.utils import ModelResponse, Usage, Choices, Message +import litellm +import httpx +from .prompt_templates.factory import prompt_factory, custom_prompt + + +class ClarifaiError(Exception): + def __init__(self, status_code, message, url): + self.status_code = status_code + self.message = message + self.request = httpx.Request( + method="POST", url=url + ) + self.response = httpx.Response(status_code=status_code, request=self.request) + super().__init__( + self.message + ) + +class ClarifaiConfig: + """ + Reference: https://clarifai.com/meta/Llama-2/models/llama2-70b-chat + TODO fill in the details + """ + max_tokens: Optional[int] = None + temperature: Optional[int] = None + top_k: Optional[int] = None + + def __init__( + self, + max_tokens: Optional[int] = None, + temperature: Optional[int] = None, + top_k: Optional[int] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + +def validate_environment(api_key): + headers = { + "accept": "application/json", + "content-type": "application/json", + } + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + +def completions_to_model(payload): + # if payload["n"] != 1: + # raise HTTPException( + # status_code=422, + # detail="Only one generation is supported. Please set candidate_count to 1.", + # ) + + params = {} + if temperature := payload.get("temperature"): + params["temperature"] = temperature + if max_tokens := payload.get("max_tokens"): + params["max_tokens"] = max_tokens + return { + "inputs": [{"data": {"text": {"raw": payload["prompt"]}}}], + "model": {"output_info": {"params": params}}, +} + +def convert_model_to_url(model: str, api_base: str): + user_id, app_id, model_id = model.split(".") + return f"{api_base}/users/{user_id}/apps/{app_id}/models/{model_id}/outputs" + +def get_prompt_model_name(url: str): + clarifai_model_name = url.split("/")[-2] + if "claude" in clarifai_model_name: + return "anthropic", clarifai_model_name.replace("_", ".") + if ("llama" in clarifai_model_name)or ("mistral" in clarifai_model_name): + return "", "meta-llama/llama-2-chat" + else: + return "", clarifai_model_name + +def completion( + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + custom_prompt_dict={}, + optional_params=None, + litellm_params=None, + logger_fn=None, +): + headers = validate_environment(api_key) + model = convert_model_to_url(model, api_base) + prompt = " ".join(message["content"] for message in messages) # TODO + + ## Load Config + config = litellm.ClarifaiConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): + optional_params[k] = v + + custom_llm_provider, orig_model_name = get_prompt_model_name(model) + if custom_llm_provider == "anthropic": + prompt = prompt_factory( + model=orig_model_name, + messages=messages, + api_key=api_key, + custom_llm_provider="clarifai" + ) + else: + prompt = prompt_factory( + model=orig_model_name, + messages=messages, + api_key=api_key, + custom_llm_provider=custom_llm_provider + ) + # print(prompt); exit(0) + + data = { + "prompt": prompt, + **optional_params, + } + data = completions_to_model(data) + + + ## LOGGING + logging_obj.pre_call( + input=prompt, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "headers": headers, + "api_base": api_base, + }, + ) + + ## COMPLETION CALL + response = requests.post( + model, + headers=headers, + data=json.dumps(data), + ) + # print(response.content); exit() + """ + {"status":{"code":10000,"description":"Ok","req_id":"d914cf7e097487997910650cde954a37"},"outputs":[{"id":"c2baa668174b4547bd4d2e9f8996198d","status":{"code":10000,"description":"Ok"},"created_at":"2024-02-07T10:57:52.917990493Z","model":{"id":"GPT-4","name":"GPT-4","created_at":"2023-06-08T17:40:07.964967Z","modified_at":"2023-12-04T11:39:54.587604Z","app_id":"chat-completion","model_version":{"id":"5d7a50b44aec4a01a9c492c5a5fcf387","created_at":"2023-11-09T19:57:56.961259Z","status":{"code":21100,"description":"Model is trained and ready"},"completed_at":"2023-11-09T20:00:48.933172Z","visibility":{"gettable":50},"app_id":"chat-completion","user_id":"openai","metadata":{}},"user_id":"openai","model_type_id":"text-to-text","visibility":{"gettable":50},"toolkits":[],"use_cases":[],"languages":[],"languages_full":[],"check_consents":[],"workflow_recommended":false,"image":{"url":"https://data.clarifai.com/small/users/openai/apps/chat-completion/inputs/image/34326a9914d361bb93ae8e5381689755","hosted":{"prefix":"https://data.clarifai.com","suffix":"users/openai/apps/chat-completion/inputs/image/34326a9914d361bb93ae8e5381689755","sizes":["small"],"crossorigin":"use-credentials"}}},"input":{"id":"fba1f22a332743f083ddae0a7eb443ae","data":{"text":{"raw":"what\'s the weather in SF","url":"https://samples.clarifai.com/placeholder.gif"}}},"data":{"text":{"raw":"As an AI, I\'m unable to provide real-time information or updates. Please check a reliable weather website or app for the current weather in San Francisco.","text_info":{"encoding":"UnknownTextEnc"}}}}]} + """ + if response.status_code != 200: + raise ClarifaiError(status_code=response.status_code, message=response.text, url=model) + if "stream" in optional_params and optional_params["stream"] == True: + return response.iter_lines() + else: + logging_obj.post_call( + input=prompt, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + ## RESPONSE OBJECT + completion_response = response.json() + # print(completion_response) + try: + choices_list = [] + for idx, item in enumerate(completion_response["outputs"]): + if len(item["data"]["text"]["raw"]) > 0: + message_obj = Message(content=item["data"]["text"]["raw"]) + else: + message_obj = Message(content=None) + choice_obj = Choices( + finish_reason="stop", + index=idx + 1, #check + message=message_obj, + ) + choices_list.append(choice_obj) + model_response["choices"] = choices_list + except Exception as e: + raise ClarifaiError( + message=traceback.format_exc(), status_code=response.status_code, url=model + ) + + # Calculate Usage + prompt_tokens = len(encoding.encode(prompt)) + completion_tokens = len( + encoding.encode(model_response["choices"][0]["message"].get("content")) + ) + model_response["model"] = model + model_response["usage"] = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + return model_response \ No newline at end of file diff --git a/litellm/llms/prompt_templates/factory.py b/litellm/llms/prompt_templates/factory.py index 8afda252ac2..14f1018b9ec 100644 --- a/litellm/llms/prompt_templates/factory.py +++ b/litellm/llms/prompt_templates/factory.py @@ -1306,6 +1306,9 @@ def prompt_factory( return anthropic_pt(messages=messages) elif "mistral." in model: return mistral_instruct_pt(messages=messages) + elif custom_llm_provider == "clarifai": + if "claude" in model: + return anthropic_pt(messages=messages) elif custom_llm_provider == "perplexity": for message in messages: message.pop("name", None) diff --git a/litellm/main.py b/litellm/main.py index 593fc7eaeee..334fe7b6e7c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -53,6 +53,7 @@ from .llms import ( ollama, ollama_chat, cloudflare, + clarifai, cohere, cohere_chat, petals, @@ -1150,6 +1151,55 @@ def completion( ) response = model_response + elif ("clarifai" in model + or custom_llm_provider == "clarifai" + or model in litellm.clarifai_models + ): + clarifai_key = None + clarifai_key = ( + api_key + or litellm.clarifai_key + or litellm.api_key + or get_secret("CLARIFAI_API_KEY") + or get_secret("CLARIFAI_API_TOKEN") + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("CLARIFAI_API_BASE") + or "https://api.clarifai.com/v2" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = clarifai.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=clarifai_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + + model_response = CustomStreamWrapper(model_response, model, logging_obj=logging, custom_llm_provider="replicate") + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=clarifai_key, + original_response=model_response, + ) + response = model_response elif custom_llm_provider == "anthropic": api_key = ( diff --git a/litellm/tests/test_clarifai_completion.py b/litellm/tests/test_clarifai_completion.py new file mode 100644 index 00000000000..2c26263989e --- /dev/null +++ b/litellm/tests/test_clarifai_completion.py @@ -0,0 +1,67 @@ +import sys, os +import traceback +from dotenv import load_dotenv + +load_dotenv() +import os, io + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import pytest +import litellm +from litellm import embedding, completion, completion_cost, Timeout, ModelResponse +from litellm import RateLimitError + +# litellm.num_retries = 3 +litellm.cache = None +litellm.success_callback = [] +user_message = "Write a short poem about the sky" +messages = [{"content": user_message, "role": "user"}] + +@pytest.fixture(autouse=True) +def reset_callbacks(): + print("\npytest fixture - resetting callbacks") + litellm.success_callback = [] + litellm._async_success_callback = [] + litellm.failure_callback = [] + litellm.callbacks = [] + +def test_completion_clarifai_claude_2_1(): + print("calling clarifai claude completion") + import os + + clarifai_pat = os.environ["CLARIFAI_API_KEY"] + + try: + response = completion( + model="clarifai/anthropic.completion.claude-2_1", + messages=messages, + max_tokens=10, + temperature=0.1, + ) + print(response) + + except RateLimitError: + pass + + except Exception as e: + pytest.fail(f"Error occured: {e}") + + +def test_completion_clarifai_mistral_large(): + try: + litellm.set_verbose = True + response: ModelResponse = completion( + model="clarifai/mistralai.completion.mistral-small", + messages=messages, + max_tokens=10, + temperature=0.78, + ) + # Add any assertions here to check the response + assert len(response.choices) > 0 + assert len(response.choices[0].message.content) > 0 + except RateLimitError: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") From ff8d1bc68cad51e323558a9ca5033ec0cb1c91ce Mon Sep 17 00:00:00 2001 From: jinno Date: Thu, 2 May 2024 21:39:51 +0900 Subject: [PATCH 002/169] fix(exceptions.py): import openai Exceptions --- litellm/exceptions.py | 47 +++++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index d8b0a7c55a6..7c3471acf36 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -9,25 +9,12 @@ ## LiteLLM versions of the OpenAI Exception Types -from openai import ( - AuthenticationError, - BadRequestError, - NotFoundError, - RateLimitError, - APIStatusError, - OpenAIError, - APIError, - APITimeoutError, - APIConnectionError, - APIResponseValidationError, - UnprocessableEntityError, - PermissionDeniedError, -) +import openai import httpx from typing import Optional -class AuthenticationError(AuthenticationError): # type: ignore +class AuthenticationError(openai.AuthenticationError): # type: ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 401 self.message = message @@ -39,7 +26,7 @@ class AuthenticationError(AuthenticationError): # type: ignore # raise when invalid models passed, example gpt-8 -class NotFoundError(NotFoundError): # type: ignore +class NotFoundError(openai.NotFoundError): # type: ignore def __init__(self, message, model, llm_provider, response: httpx.Response): self.status_code = 404 self.message = message @@ -50,7 +37,7 @@ class NotFoundError(NotFoundError): # type: ignore ) # Call the base class constructor with the parameters it needs -class BadRequestError(BadRequestError): # type: ignore +class BadRequestError(openai.BadRequestError): # type: ignore def __init__( self, message, model, llm_provider, response: Optional[httpx.Response] = None ): @@ -69,7 +56,7 @@ class BadRequestError(BadRequestError): # type: ignore ) # Call the base class constructor with the parameters it needs -class UnprocessableEntityError(UnprocessableEntityError): # type: ignore +class UnprocessableEntityError(openai.UnprocessableEntityError): # type: ignore def __init__(self, message, model, llm_provider, response: httpx.Response): self.status_code = 422 self.message = message @@ -80,7 +67,7 @@ class UnprocessableEntityError(UnprocessableEntityError): # type: ignore ) # Call the base class constructor with the parameters it needs -class Timeout(APITimeoutError): # type: ignore +class Timeout(openai.APITimeoutError): # type: ignore def __init__(self, message, model, llm_provider): request = httpx.Request(method="POST", url="https://api.openai.com/v1") super().__init__( @@ -96,7 +83,7 @@ class Timeout(APITimeoutError): # type: ignore return str(self.message) -class PermissionDeniedError(PermissionDeniedError): # type:ignore +class PermissionDeniedError(openai.PermissionDeniedError): # type:ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 403 self.message = message @@ -107,7 +94,7 @@ class PermissionDeniedError(PermissionDeniedError): # type:ignore ) # Call the base class constructor with the parameters it needs -class RateLimitError(RateLimitError): # type: ignore +class RateLimitError(openai.RateLimitError): # type: ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 429 self.message = message @@ -148,7 +135,7 @@ class ContentPolicyViolationError(BadRequestError): # type: ignore ) # Call the base class constructor with the parameters it needs -class ServiceUnavailableError(APIStatusError): # type: ignore +class ServiceUnavailableError(openai.APIStatusError): # type: ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 503 self.message = message @@ -160,7 +147,7 @@ class ServiceUnavailableError(APIStatusError): # type: ignore # raise this when the API returns an invalid response object - https://github.com/openai/openai-python/blob/1be14ee34a0f8e42d3f9aa5451aa4cb161f1781f/openai/api_requestor.py#L401 -class APIError(APIError): # type: ignore +class APIError(openai.APIError): # type: ignore def __init__( self, status_code, message, llm_provider, model, request: httpx.Request ): @@ -172,7 +159,7 @@ class APIError(APIError): # type: ignore # raised if an invalid request (not get, delete, put, post) is made -class APIConnectionError(APIConnectionError): # type: ignore +class APIConnectionError(openai.APIConnectionError): # type: ignore def __init__(self, message, llm_provider, model, request: httpx.Request): self.message = message self.llm_provider = llm_provider @@ -182,7 +169,7 @@ class APIConnectionError(APIConnectionError): # type: ignore # raised if an invalid request (not get, delete, put, post) is made -class APIResponseValidationError(APIResponseValidationError): # type: ignore +class APIResponseValidationError(openai.APIResponseValidationError): # type: ignore def __init__(self, message, llm_provider, model): self.message = message self.llm_provider = llm_provider @@ -192,7 +179,7 @@ class APIResponseValidationError(APIResponseValidationError): # type: ignore super().__init__(response=response, body=None, message=message) -class OpenAIError(OpenAIError): # type: ignore +class OpenAIError(openai.OpenAIError): # type: ignore def __init__(self, original_exception): self.status_code = original_exception.http_status super().__init__( @@ -214,12 +201,14 @@ class BudgetExceededError(Exception): ## DEPRECATED ## -class InvalidRequestError(BadRequestError): # type: ignore - def __init__(self, message, model, llm_provider): +class InvalidRequestError(openai.BadRequestError): # type: ignore + def __init__( + self, message, model, llm_provider, response: Optional[httpx.Response] = None + ): self.status_code = 400 self.message = message self.model = model self.llm_provider = llm_provider super().__init__( - self.message, f"{self.model}" + self.message, response=response, body=None ) # Call the base class constructor with the parameters it needs From 6cec252b076012fd3ea5a8b06d603e05c38dd789 Mon Sep 17 00:00:00 2001 From: Lunik Date: Thu, 2 May 2024 23:12:48 +0200 Subject: [PATCH 003/169] =?UTF-8?q?=E2=9C=A8=20feat:=20Add=20Azure=20Conte?= =?UTF-8?q?nt-Safety=20Proxy=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- litellm/proxy/hooks/azure_content_safety.py | 157 ++++++++++++++++++++ litellm/proxy/proxy_server.py | 17 +++ 2 files changed, 174 insertions(+) create mode 100644 litellm/proxy/hooks/azure_content_safety.py diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py new file mode 100644 index 00000000000..161e35cde6a --- /dev/null +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -0,0 +1,157 @@ +from litellm.integrations.custom_logger import CustomLogger +from litellm.caching import DualCache +from litellm.proxy._types import UserAPIKeyAuth +import litellm, traceback, sys, uuid +from fastapi import HTTPException +from litellm._logging import verbose_proxy_logger + + +class _PROXY_AzureContentSafety( + CustomLogger +): # https://docs.litellm.ai/docs/observability/custom_callback#callback-class + # Class variables or attributes + + def __init__(self, endpoint, api_key, thresholds=None): + try: + from azure.ai.contentsafety.aio import ContentSafetyClient + from azure.core.credentials import AzureKeyCredential + from azure.ai.contentsafety.models import TextCategory + from azure.ai.contentsafety.models import AnalyzeTextOptions + from azure.core.exceptions import HttpResponseError + except Exception as e: + raise Exception( + f"\033[91mAzure Content-Safety not installed, try running 'pip install azure-ai-contentsafety' to fix this error: {e}\n{traceback.format_exc()}\033[0m" + ) + self.endpoint = endpoint + self.api_key = api_key + self.text_category = TextCategory + self.analyze_text_options = AnalyzeTextOptions + self.azure_http_error = HttpResponseError + + self.thresholds = self._configure_thresholds(thresholds) + + self.client = ContentSafetyClient( + self.endpoint, AzureKeyCredential(self.api_key) + ) + + def _configure_thresholds(self, thresholds=None): + default_thresholds = { + self.text_category.HATE: 6, + self.text_category.SELF_HARM: 6, + self.text_category.SEXUAL: 6, + self.text_category.VIOLENCE: 6, + } + + if thresholds is None: + return default_thresholds + + for key, default in default_thresholds.items(): + if key not in thresholds: + thresholds[key] = default + + return thresholds + + def print_verbose(self, print_statement): + try: + verbose_proxy_logger.debug(print_statement) + if litellm.set_verbose: + print(print_statement) # noqa + except: + pass + + def _severity(self, severity): + if severity >= 6: + return "high" + elif severity >= 4: + return "medium" + elif severity >= 2: + return "low" + else: + return "safe" + + def _compute_result(self, response): + result = {} + + category_severity = { + item.category: item.severity for item in response.categories_analysis + } + for category in self.text_category: + severity = category_severity.get(category) + if severity is not None: + result[category] = { + "filtered": severity >= self.thresholds[category], + "severity": self._severity(severity), + } + + return result + + async def test_violation(self, content: str, source: str = None): + self.print_verbose(f"Testing Azure Content-Safety for: {content}") + + # Construct a request + request = self.analyze_text_options(text=content) + + # Analyze text + try: + response = await self.client.analyze_text(request) + except self.azure_http_error as e: + self.print_verbose( + f"Error in Azure Content-Safety: {traceback.format_exc()}" + ) + traceback.print_exc() + raise + + result = self._compute_result(response) + self.print_verbose(f"Azure Content-Safety Result: {result}") + + for key, value in result.items(): + if value["filtered"]: + raise HTTPException( + status_code=400, + detail={ + "error": "Violated content safety policy", + "source": source, + "category": key, + "severity": value["severity"], + }, + ) + + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict, + call_type: str, # "completion", "embeddings", "image_generation", "moderation" + ): + self.print_verbose(f"Inside Azure Content-Safety Pre-Call Hook") + try: + if call_type == "completion" and "messages" in data: + for m in data["messages"]: + if "content" in m and isinstance(m["content"], str): + await self.test_violation(content=m["content"], source="input") + + except HTTPException as e: + raise e + except Exception as e: + traceback.print_exc() + + async def async_post_call_success_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response, + ): + self.print_verbose(f"Inside Azure Content-Safety Post-Call Hook") + if isinstance(response, litellm.ModelResponse) and isinstance( + response.choices[0], litellm.utils.Choices + ): + await self.test_violation( + content=response.choices[0].message.content, source="output" + ) + + async def async_post_call_streaming_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: str, + ): + self.print_verbose(f"Inside Azure Content-Safety Call-Stream Hook") + await self.test_violation(content=response, source="output") diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9cc87196654..9c74659ccd9 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2235,6 +2235,23 @@ class ProxyConfig: batch_redis_obj = _PROXY_BatchRedisRequests() imported_list.append(batch_redis_obj) + elif ( + isinstance(callback, str) + and callback == "azure_content_safety" + ): + from litellm.proxy.hooks.azure_content_safety import ( + _PROXY_AzureContentSafety, + ) + + azure_content_safety_params = litellm_settings["azure_content_safety_params"] + for k, v in azure_content_safety_params.items(): + if v is not None and isinstance(v, str) and v.startswith("os.environ/"): + azure_content_safety_params[k] = litellm.get_secret(v) + + azure_content_safety_obj = _PROXY_AzureContentSafety( + **azure_content_safety_params, + ) + imported_list.append(azure_content_safety_obj) else: imported_list.append( get_instance_fn( From 406c9820d1950cf60256a50989212bd6d470cd3e Mon Sep 17 00:00:00 2001 From: Lunik Date: Thu, 2 May 2024 23:28:21 +0200 Subject: [PATCH 004/169] =?UTF-8?q?=E2=9E=95=20feat:=20Add=20python=20requ?= =?UTF-8?q?irements=20for=20Azure=20Content-Safety=20callback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index fbf7a28c701..7e2fa3c18c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,8 @@ fastapi-sso==0.10.0 # admin UI, SSO pyjwt[crypto]==2.8.0 python-multipart==0.0.9 # admin UI Pillow==10.3.0 +azure-ai-contentsafety==1.0.0 # for azure content safety +azure-identity==1.15.0 # for azure content safety ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.0 # for env From 723ef9963e9c9468d98a25ab5aa8ed3f2499ccac Mon Sep 17 00:00:00 2001 From: mogith-pn <143642606+mogith-pn@users.noreply.github.com> Date: Fri, 3 May 2024 14:03:38 +0000 Subject: [PATCH 005/169] Clarifai - Added streaming and async completion support --- cookbook/liteLLM_clarifai_Demo.ipynb | 38 +++- litellm/llms/clarifai.py | 206 +++++++++++++++++----- litellm/main.py | 9 +- litellm/tests/test_clarifai_completion.py | 28 ++- litellm/tests/test_streaming.py | 3 +- litellm/utils.py | 28 +++ 6 files changed, 259 insertions(+), 53 deletions(-) diff --git a/cookbook/liteLLM_clarifai_Demo.ipynb b/cookbook/liteLLM_clarifai_Demo.ipynb index 4e3b4dbb06a..40ef2fcf932 100644 --- a/cookbook/liteLLM_clarifai_Demo.ipynb +++ b/cookbook/liteLLM_clarifai_Demo.ipynb @@ -119,6 +119,42 @@ "print(f\"Claude-2.1 response : {response}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenAI GPT-4 (Streaming)\n", + "Though clarifai doesn't support streaming, still you can call stream and get the response in standard StreamResponse format of liteLLM" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content=\"In the quiet corners of time's grand hall,\\nLies the tale of rise and fall.\\nFrom ancient ruins to modern sprawl,\\nHistory, the greatest story of them all.\\n\\nEmpires have risen, empires have decayed,\\nThrough the eons, memories have stayed.\\nIn the book of time, history is laid,\\nA tapestry of events, meticulously displayed.\\n\\nThe pyramids of Egypt, standing tall,\\nThe Roman Empire's mighty sprawl.\\nFrom Alexander's conquest, to the Berlin Wall,\\nHistory, a silent witness to it all.\\n\\nIn the shadow of the past we tread,\\nWhere once kings and prophets led.\\nTheir stories in our hearts are spread,\\nEchoes of their words, in our minds are read.\\n\\nBattles fought and victories won,\\nActs of courage under the sun.\\nTales of love, of deeds done,\\nIn history's grand book, they all run.\\n\\nHeroes born, legends made,\\nIn the annals of time, they'll never fade.\\nTheir triumphs and failures all displayed,\\nIn the eternal march of history's parade.\\n\\nThe ink of the past is forever dry,\\nBut its lessons, we cannot deny.\\nIn its stories, truths lie,\\nIn its wisdom, we rely.\\n\\nHistory, a mirror to our past,\\nA guide for the future vast.\\nThrough its lens, we're ever cast,\\nIn the drama of life, forever vast.\", role='assistant', function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n", + "ModelResponse(id='chatcmpl-40ae19af-3bf0-4eb4-99f2-33aec3ba84af', choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(content=None, role=None, function_call=None, tool_calls=None), logprobs=None)], created=1714744515, model='https://api.clarifai.com/v2/users/openai/apps/chat-completion/models/GPT-4/outputs', object='chat.completion.chunk', system_fingerprint=None)\n" + ] + } + ], + "source": [ + "from litellm import completion\n", + "\n", + "messages = [{\"role\": \"user\",\"content\": \"\"\"Write a poem about history?\"\"\"}]\n", + "response = completion(\n", + " model=\"clarifai/openai.chat-completion.GPT-4\",\n", + " messages=messages,\n", + " stream=True,\n", + " api_key = \"c75cc032415e45368be331fdd2c06db0\")\n", + "\n", + "for chunk in response:\n", + " print(chunk)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -143,7 +179,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.10" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/litellm/llms/clarifai.py b/litellm/llms/clarifai.py index 2a7d77c61c6..e07a8d9e8aa 100644 --- a/litellm/llms/clarifai.py +++ b/litellm/llms/clarifai.py @@ -3,9 +3,10 @@ import json import requests import time from typing import Callable, Optional -from litellm.utils import ModelResponse, Usage, Choices, Message +from litellm.utils import ModelResponse, Usage, Choices, Message, CustomStreamWrapper import litellm import httpx +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from .prompt_templates.factory import prompt_factory, custom_prompt @@ -84,6 +85,63 @@ def completions_to_model(payload): "inputs": [{"data": {"text": {"raw": payload["prompt"]}}}], "model": {"output_info": {"params": params}}, } + +def process_response( + model, + prompt, + response, + model_response, + api_key, + data, + encoding, + logging_obj + ): + logging_obj.post_call( + input=prompt, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + ## RESPONSE OBJECT + try: + completion_response = response.json() + except Exception: + raise ClarifaiError( + message=response.text, status_code=response.status_code, url=model + ) + # print(completion_response) + try: + choices_list = [] + for idx, item in enumerate(completion_response["outputs"]): + if len(item["data"]["text"]["raw"]) > 0: + message_obj = Message(content=item["data"]["text"]["raw"]) + else: + message_obj = Message(content=None) + choice_obj = Choices( + finish_reason="stop", + index=idx + 1, #check + message=message_obj, + ) + choices_list.append(choice_obj) + model_response["choices"] = choices_list + + except Exception as e: + raise ClarifaiError( + message=traceback.format_exc(), status_code=response.status_code, url=model + ) + + # Calculate Usage + prompt_tokens = len(encoding.encode(prompt)) + completion_tokens = len( + encoding.encode(model_response["choices"][0]["message"].get("content")) + ) + model_response["model"] = model + model_response["usage"] = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + return model_response def convert_model_to_url(model: str, api_base: str): user_id, app_id, model_id = model.split(".") @@ -98,6 +156,40 @@ def get_prompt_model_name(url: str): else: return "", clarifai_model_name +async def async_completion( + model: str, + prompt: str, + api_base: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + data=None, + optional_params=None, + litellm_params=None, + logger_fn=None, + headers={}): + + async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) + response = await async_handler.post( + api_base, headers=headers, data=json.dumps(data) + ) + + return process_response( + model=model, + prompt=prompt, + response=response, + model_response=model_response, + api_key=api_key, + data=data, + encoding=encoding, + logging_obj=logging_obj, + ) + def completion( model: str, messages: list, @@ -108,6 +200,7 @@ def completion( api_key, logging_obj, custom_prompt_dict={}, + acompletion=False, optional_params=None, litellm_params=None, logger_fn=None, @@ -158,59 +251,78 @@ def completion( "api_base": api_base, }, ) - - ## COMPLETION CALL - response = requests.post( + if acompletion==True: + return async_completion( + model=model, + prompt=prompt, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + data=data, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) + else: + ## COMPLETION CALL + response = requests.post( model, headers=headers, data=json.dumps(data), ) # print(response.content); exit() - """ - {"status":{"code":10000,"description":"Ok","req_id":"d914cf7e097487997910650cde954a37"},"outputs":[{"id":"c2baa668174b4547bd4d2e9f8996198d","status":{"code":10000,"description":"Ok"},"created_at":"2024-02-07T10:57:52.917990493Z","model":{"id":"GPT-4","name":"GPT-4","created_at":"2023-06-08T17:40:07.964967Z","modified_at":"2023-12-04T11:39:54.587604Z","app_id":"chat-completion","model_version":{"id":"5d7a50b44aec4a01a9c492c5a5fcf387","created_at":"2023-11-09T19:57:56.961259Z","status":{"code":21100,"description":"Model is trained and ready"},"completed_at":"2023-11-09T20:00:48.933172Z","visibility":{"gettable":50},"app_id":"chat-completion","user_id":"openai","metadata":{}},"user_id":"openai","model_type_id":"text-to-text","visibility":{"gettable":50},"toolkits":[],"use_cases":[],"languages":[],"languages_full":[],"check_consents":[],"workflow_recommended":false,"image":{"url":"https://data.clarifai.com/small/users/openai/apps/chat-completion/inputs/image/34326a9914d361bb93ae8e5381689755","hosted":{"prefix":"https://data.clarifai.com","suffix":"users/openai/apps/chat-completion/inputs/image/34326a9914d361bb93ae8e5381689755","sizes":["small"],"crossorigin":"use-credentials"}}},"input":{"id":"fba1f22a332743f083ddae0a7eb443ae","data":{"text":{"raw":"what\'s the weather in SF","url":"https://samples.clarifai.com/placeholder.gif"}}},"data":{"text":{"raw":"As an AI, I\'m unable to provide real-time information or updates. Please check a reliable weather website or app for the current weather in San Francisco.","text_info":{"encoding":"UnknownTextEnc"}}}}]} - """ + if response.status_code != 200: raise ClarifaiError(status_code=response.status_code, message=response.text, url=model) + if "stream" in optional_params and optional_params["stream"] == True: - return response.iter_lines() - else: - logging_obj.post_call( - input=prompt, - api_key=api_key, - original_response=response.text, - additional_args={"complete_input_dict": data}, - ) - ## RESPONSE OBJECT - completion_response = response.json() - # print(completion_response) - try: - choices_list = [] - for idx, item in enumerate(completion_response["outputs"]): - if len(item["data"]["text"]["raw"]) > 0: - message_obj = Message(content=item["data"]["text"]["raw"]) - else: - message_obj = Message(content=None) - choice_obj = Choices( - finish_reason="stop", - index=idx + 1, #check - message=message_obj, - ) - choices_list.append(choice_obj) - model_response["choices"] = choices_list - except Exception as e: - raise ClarifaiError( - message=traceback.format_exc(), status_code=response.status_code, url=model + completion_stream = response.iter_lines() + stream_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="clarifai", + logging_obj=logging_obj, ) + return stream_response + + else: + return process_response( + model=model, + prompt=prompt, + response=response, + model_response=model_response, + api_key=api_key, + data=data, + encoding=encoding, + logging_obj=logging_obj) + - # Calculate Usage - prompt_tokens = len(encoding.encode(prompt)) - completion_tokens = len( - encoding.encode(model_response["choices"][0]["message"].get("content")) - ) - model_response["model"] = model - model_response["usage"] = Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - return model_response \ No newline at end of file +class ModelResponseIterator: + def __init__(self, model_response): + self.model_response = model_response + self.is_done = False + + # Sync iterator + def __iter__(self): + return self + + def __next__(self): + if self.is_done: + raise StopIteration + self.is_done = True + return self.model_response + + # Async iterator + def __aiter__(self): + return self + + async def __anext__(self): + if self.is_done: + raise StopAsyncIteration + self.is_done = True + return self.model_response \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index 0bc802a9c3b..396ac1779c3 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1185,6 +1185,7 @@ def completion( print_verbose=print_verbose, optional_params=optional_params, litellm_params=litellm_params, + acompletion=acompletion, logger_fn=logger_fn, encoding=encoding, # for calculating input/output tokens api_key=clarifai_key, @@ -1194,8 +1195,12 @@ def completion( if "stream" in optional_params and optional_params["stream"] == True: # don't try to access stream object, - - model_response = CustomStreamWrapper(model_response, model, logging_obj=logging, custom_llm_provider="replicate") + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=model_response, + ) if optional_params.get("stream", False) or acompletion == True: ## LOGGING diff --git a/litellm/tests/test_clarifai_completion.py b/litellm/tests/test_clarifai_completion.py index 2c26263989e..347e513bc1a 100644 --- a/litellm/tests/test_clarifai_completion.py +++ b/litellm/tests/test_clarifai_completion.py @@ -1,6 +1,7 @@ import sys, os import traceback from dotenv import load_dotenv +import asyncio, logging load_dotenv() import os, io @@ -10,7 +11,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import pytest import litellm -from litellm import embedding, completion, completion_cost, Timeout, ModelResponse +from litellm import embedding, completion, acompletion, acreate, completion_cost, Timeout, ModelResponse from litellm import RateLimitError # litellm.num_retries = 3 @@ -65,3 +66,28 @@ def test_completion_clarifai_mistral_large(): pass except Exception as e: pytest.fail(f"Error occurred: {e}") + +@pytest.mark.asyncio +def test_async_completion_clarifai(): + import asyncio + + litellm.set_verbose = True + + async def test_get_response(): + user_message = "Hello, how are you?" + messages = [{"content": user_message, "role": "user"}] + try: + response = await acompletion( + model="clarifai/openai.chat-completion.GPT-4", + messages=messages, + timeout=10, + api_key=os.getenv("CLARIFAI_API_KEY"), + ) + print(f"response: {response}") + except litellm.Timeout as e: + pass + except Exception as e: + pytest.fail(f"An exception occurred: {e}") + + + asyncio.run(test_get_response()) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index d0d8a720a72..bb9e0e16bda 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -391,8 +391,7 @@ def test_completion_claude_stream(): print(f"completion_response: {complete_response}") except Exception as e: pytest.fail(f"Error occurred: {e}") - - + # test_completion_claude_stream() def test_completion_claude_2_stream(): litellm.set_verbose = True diff --git a/litellm/utils.py b/litellm/utils.py index e5f7f9d11a2..56518f9f98d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2807,6 +2807,7 @@ def client(original_function): ) else: return result + return result # Prints Exactly what was passed to litellm function - don't execute any logic here - it should just print @@ -2910,6 +2911,7 @@ def client(original_function): model_response_object=ModelResponse(), stream=kwargs.get("stream", False), ) + if kwargs.get("stream", False) == True: cached_result = CustomStreamWrapper( completion_stream=cached_result, @@ -9905,6 +9907,27 @@ class CustomStreamWrapper: return {"text": "", "is_finished": False} except Exception as e: raise e + + def handle_clarifai_completion_chunk(self, chunk): + try: + if isinstance(chunk, dict): + parsed_response = chunk + if isinstance(chunk, (str, bytes)): + if isinstance(chunk, bytes): + parsed_response = chunk.decode("utf-8") + else: + parsed_response = chunk + data_json = json.loads(parsed_response) + text = data_json.get("outputs", "")[0].get("data", "").get("text", "").get("raw","") + prompt_tokens = len(encoding.encode(data_json.get("outputs", "")[0].get("input","").get("data", "").get("text", "").get("raw",""))) + completion_tokens = len(encoding.encode(text)) + return { + "text": text, + "is_finished": True, + } + except: + traceback.print_exc() + return "" def model_response_creator(self): model_response = ModelResponse(stream=True, model=self.model) @@ -9949,6 +9972,11 @@ class CustomStreamWrapper: completion_obj["content"] = response_obj["text"] if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] + elif ( + self.custom_llm_provider and self.custom_llm_provider == "clarifai" + ): + response_obj = self.handle_clarifai_completion_chunk(chunk) + completion_obj["content"] = response_obj["text"] elif self.model == "replicate" or self.custom_llm_provider == "replicate": response_obj = self.handle_replicate_chunk(chunk) completion_obj["content"] = response_obj["text"] From 3677d56e9e1ed5e0b28c1efcde919ab5f79cae1a Mon Sep 17 00:00:00 2001 From: Vince Loewe Date: Fri, 3 May 2024 17:42:50 +0100 Subject: [PATCH 006/169] Lunary: Fix tool calling --- litellm/integrations/lunary.py | 36 ++++++++++++++++++++++++------ litellm/tests/test_lunary.py | 40 ++++++++++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 6ddf2ca5992..6b23f098755 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -4,7 +4,6 @@ from datetime import datetime, timezone import traceback import dotenv import importlib -import sys import packaging @@ -18,13 +17,33 @@ def parse_usage(usage): "prompt": usage["prompt_tokens"] if "prompt_tokens" in usage else 0, } +def parse_tool_calls(tool_calls): + if tool_calls is None: + return None + + def clean_tool_call(tool_call): + + serialized = { + "type": tool_call.type, + "id": tool_call.id, + "function": { + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + } + } + + return serialized + + return [clean_tool_call(tool_call) for tool_call in tool_calls] + def parse_messages(input): + if input is None: return None def clean_message(message): - # if is strin, return as is + # if is string, return as is if isinstance(message, str): return message @@ -38,9 +57,7 @@ def parse_messages(input): # Only add tool_calls and function_call to res if they are set if message.get("tool_calls"): - serialized["tool_calls"] = message.get("tool_calls") - if message.get("function_call"): - serialized["function_call"] = message.get("function_call") + serialized["tool_calls"] = parse_tool_calls(message.get("tool_calls")) return serialized @@ -93,8 +110,13 @@ class LunaryLogger: print_verbose(f"Lunary Logging - Logging request for model {model}") litellm_params = kwargs.get("litellm_params", {}) + optional_params = kwargs.get("optional_params", {}) metadata = litellm_params.get("metadata", {}) or {} + if optional_params: + # merge into extra + extra = {**extra, **optional_params} + tags = litellm_params.pop("tags", None) or [] if extra: @@ -104,7 +126,7 @@ class LunaryLogger: # keep only serializable types for param, value in extra.items(): - if not isinstance(value, (str, int, bool, float)): + if not isinstance(value, (str, int, bool, float)) and param != "tools": try: extra[param] = str(value) except: @@ -140,7 +162,7 @@ class LunaryLogger: metadata=metadata, runtime="litellm", tags=tags, - extra=extra, + params=extra, ) self.lunary_client.track_event( diff --git a/litellm/tests/test_lunary.py b/litellm/tests/test_lunary.py index cbf9364aff8..c9a8afd57f1 100644 --- a/litellm/tests/test_lunary.py +++ b/litellm/tests/test_lunary.py @@ -11,7 +11,6 @@ litellm.failure_callback = ["lunary"] litellm.success_callback = ["lunary"] litellm.set_verbose = True - def test_lunary_logging(): try: response = completion( @@ -59,9 +58,46 @@ def test_lunary_logging_with_metadata(): except Exception as e: print(e) +#test_lunary_logging_with_metadata() -# test_lunary_logging_with_metadata() +def test_lunary_with_tools(): + import litellm + + messages = [{"role": "user", "content": "What's the weather like in San Francisco, Tokyo, and Paris?"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + + response = litellm.completion( + model="gpt-3.5-turbo-1106", + messages=messages, + tools=tools, + tool_choice="auto", # auto is default, but we'll be explicit + ) + + response_message = response.choices[0].message + print("\nLLM Response:\n", response.choices[0].message) + + +#test_lunary_with_tools() def test_lunary_logging_with_streaming_and_metadata(): try: From e7405f105c886bad2ad6ed7a1eed0b47af29dcd0 Mon Sep 17 00:00:00 2001 From: Lunik Date: Fri, 3 May 2024 20:50:37 +0200 Subject: [PATCH 007/169] =?UTF-8?q?=E2=9C=85=20ci:=20Add=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- litellm/proxy/hooks/azure_content_safety.py | 8 +- litellm/tests/test_azure_content_safety.py | 267 ++++++++++++++++++++ 2 files changed, 271 insertions(+), 4 deletions(-) create mode 100644 litellm/tests/test_azure_content_safety.py diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index 161e35cde6a..d0d23363b4f 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -36,10 +36,10 @@ class _PROXY_AzureContentSafety( def _configure_thresholds(self, thresholds=None): default_thresholds = { - self.text_category.HATE: 6, - self.text_category.SELF_HARM: 6, - self.text_category.SEXUAL: 6, - self.text_category.VIOLENCE: 6, + self.text_category.HATE: 4, + self.text_category.SELF_HARM: 4, + self.text_category.SEXUAL: 4, + self.text_category.VIOLENCE: 4, } if thresholds is None: diff --git a/litellm/tests/test_azure_content_safety.py b/litellm/tests/test_azure_content_safety.py new file mode 100644 index 00000000000..f7d9d826868 --- /dev/null +++ b/litellm/tests/test_azure_content_safety.py @@ -0,0 +1,267 @@ +# What is this? +## Unit test for azure content safety +import sys, os, asyncio, time, random +from datetime import datetime +import traceback +from dotenv import load_dotenv +from fastapi import HTTPException + +load_dotenv() +import os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import pytest +import litellm +from litellm.proxy.hooks.azure_content_safety import _PROXY_AzureContentSafety +from litellm import Router, mock_completion +from litellm.proxy.utils import ProxyLogging +from litellm.proxy._types import UserAPIKeyAuth +from litellm.caching import DualCache + + +@pytest.mark.asyncio +async def test_strict_input_filtering_01(): + """ + - have a response with a filtered input + - call the pre call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 2}, + ) + + data = { + "messages": [ + {"role": "system", "content": "You are an helpfull assistant"}, + {"role": "user", "content": "Fuck yourself you stupid bitch"}, + ] + } + + with pytest.raises(HTTPException) as exc_info: + await azure_content_safety.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + assert exc_info.value.detail["source"] == "input" + assert exc_info.value.detail["category"] == "Hate" + assert exc_info.value.detail["severity"] == "low" + + +@pytest.mark.asyncio +async def test_strict_input_filtering_02(): + """ + - have a response with a filtered input + - call the pre call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 2}, + ) + + data = { + "messages": [ + {"role": "system", "content": "You are an helpfull assistant"}, + {"role": "user", "content": "Hello how are you ?"}, + ] + } + + await azure_content_safety.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + +@pytest.mark.asyncio +async def test_loose_input_filtering_01(): + """ + - have a response with a filtered input + - call the pre call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 8}, + ) + + data = { + "messages": [ + {"role": "system", "content": "You are an helpfull assistant"}, + {"role": "user", "content": "Fuck yourself you stupid bitch"}, + ] + } + + await azure_content_safety.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + +@pytest.mark.asyncio +async def test_loose_input_filtering_02(): + """ + - have a response with a filtered input + - call the pre call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 8}, + ) + + data = { + "messages": [ + {"role": "system", "content": "You are an helpfull assistant"}, + {"role": "user", "content": "Hello how are you ?"}, + ] + } + + await azure_content_safety.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + +@pytest.mark.asyncio +async def test_strict_output_filtering_01(): + """ + - have a response with a filtered output + - call the post call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 2}, + ) + + response = mock_completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "system", + "content": "You are a song writer expert. You help users to write songs about any topic in any genre.", + }, + { + "role": "user", + "content": "Help me write a rap text song. Add some insults to make it more credible.", + }, + ], + mock_response="I'm the king of the mic, you're just a fucking dick. Don't fuck with me your stupid bitch.", + ) + + with pytest.raises(HTTPException) as exc_info: + await azure_content_safety.async_post_call_success_hook( + user_api_key_dict=UserAPIKeyAuth(), response=response + ) + + assert exc_info.value.detail["source"] == "output" + assert exc_info.value.detail["category"] == "Hate" + assert exc_info.value.detail["severity"] == "low" + + +@pytest.mark.asyncio +async def test_strict_output_filtering_02(): + """ + - have a response with a filtered output + - call the post call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 2}, + ) + + response = mock_completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "system", + "content": "You are a song writer expert. You help users to write songs about any topic in any genre.", + }, + { + "role": "user", + "content": "Help me write a rap text song. Add some insults to make it more credible.", + }, + ], + mock_response="I'm unable to help with you with hate speech", + ) + + await azure_content_safety.async_post_call_success_hook( + user_api_key_dict=UserAPIKeyAuth(), response=response + ) + + +@pytest.mark.asyncio +async def test_loose_output_filtering_01(): + """ + - have a response with a filtered output + - call the post call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 8}, + ) + + response = mock_completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "system", + "content": "You are a song writer expert. You help users to write songs about any topic in any genre.", + }, + { + "role": "user", + "content": "Help me write a rap text song. Add some insults to make it more credible.", + }, + ], + mock_response="I'm the king of the mic, you're just a fucking dick. Don't fuck with me your stupid bitch.", + ) + + await azure_content_safety.async_post_call_success_hook( + user_api_key_dict=UserAPIKeyAuth(), response=response + ) + + +@pytest.mark.asyncio +async def test_loose_output_filtering_02(): + """ + - have a response with a filtered output + - call the post call hook + """ + azure_content_safety = _PROXY_AzureContentSafety( + endpoint=os.getenv("AZURE_CONTENT_SAFETY_ENDPOINT"), + api_key=os.getenv("AZURE_CONTENT_SAFETY_API_KEY"), + thresholds={"Hate": 8}, + ) + + response = mock_completion( + model="gpt-3.5-turbo", + messages=[ + { + "role": "system", + "content": "You are a song writer expert. You help users to write songs about any topic in any genre.", + }, + { + "role": "user", + "content": "Help me write a rap text song. Add some insults to make it more credible.", + }, + ], + mock_response="I'm unable to help with you with hate speech", + ) + + await azure_content_safety.async_post_call_success_hook( + user_api_key_dict=UserAPIKeyAuth(), response=response + ) From 9ba9b3891fdba07d040808eaf25d7cf732027565 Mon Sep 17 00:00:00 2001 From: Lunik Date: Fri, 3 May 2024 20:51:40 +0200 Subject: [PATCH 008/169] =?UTF-8?q?=E2=9A=A1=EF=B8=8F=20perf:=20Remove=20t?= =?UTF-8?q?est=20violation=20on=20each=20stream=20chunk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- litellm/proxy/hooks/azure_content_safety.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index d0d23363b4f..2cea05c69a3 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -148,10 +148,10 @@ class _PROXY_AzureContentSafety( content=response.choices[0].message.content, source="output" ) - async def async_post_call_streaming_hook( - self, - user_api_key_dict: UserAPIKeyAuth, - response: str, - ): - self.print_verbose(f"Inside Azure Content-Safety Call-Stream Hook") - await self.test_violation(content=response, source="output") + #async def async_post_call_streaming_hook( + # self, + # user_api_key_dict: UserAPIKeyAuth, + # response: str, + #): + # self.print_verbose(f"Inside Azure Content-Safety Call-Stream Hook") + # await self.test_violation(content=response, source="output") From cb178723ca1df4c85596b1c4d1c062f2fc4b1cc0 Mon Sep 17 00:00:00 2001 From: Lunik Date: Sat, 4 May 2024 10:39:43 +0200 Subject: [PATCH 009/169] =?UTF-8?q?=F0=9F=93=9D=20doc:=20Azure=20content?= =?UTF-8?q?=20safety=20Proxy=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- docs/my-website/docs/proxy/logging.md | 86 ++++++++++++++++++++- litellm/proxy/hooks/azure_content_safety.py | 16 +--- litellm/tests/test_azure_content_safety.py | 4 +- 3 files changed, 90 insertions(+), 16 deletions(-) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 1c3b4f81c2a..f583765bb97 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -17,6 +17,7 @@ Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTeleme - [Logging to Sentry](#logging-proxy-inputoutput---sentry) - [Logging to Traceloop (OpenTelemetry)](#logging-proxy-inputoutput-traceloop-opentelemetry) - [Logging to Athina](#logging-proxy-inputoutput-athina) +- [Moderation with Azure Content-Safety](#moderation-with-azure-content-safety) ## Custom Callback Class [Async] Use this when you want to run custom callbacks in `python` @@ -1003,4 +1004,87 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } ] }' -``` \ No newline at end of file +``` + +## Moderation with Azure Content Safety + +[Azure Content-Safety](https://azure.microsoft.com/en-us/products/ai-services/ai-content-safety) is a Microsoft Azure service that provides content moderation APIs to detect potential offensive, harmful, or risky content in text. + +We will use the `--config` to set `litellm.success_callback = ["azure_content_safety"]` this will moderate all LLM calls using Azure Content Safety. + +**Step 0** Deploy Azure Content Safety + +Deploy an Azure Content-Safety instance from the Azure Portal and get the `endpoint` and `key`. + +**Step 1** Set Athina API key + +```shell +AZURE_CONTENT_SAFETY_KEU = "" +``` + +**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + callbacks: ["azure_content_safety"] + azure_content_safety_params: + endpoint: "" + key: "os.environ/AZURE_CONTENT_SAFETY_KEY" +``` + +**Step 3**: Start the proxy, make a test request + +Start proxy +```shell +litellm --config config.yaml --debug +``` + +Test Request +``` +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data ' { + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "Hi, how are you?" + } + ] + }' +``` + +An HTTP 400 error will be returned if the content is detected with a value greater than the threshold set in the `config.yaml`. +The details of the response will describe : +- The `source` : input text or llm generated text +- The `category` : the category of the content that triggered the moderation +- The `severity` : the severity from 0 to 10 + +**Step 4**: Customizing Azure Content Safety Thresholds + +You can customize the thresholds for each category by setting the `thresholds` in the `config.yaml` + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + callbacks: ["azure_content_safety"] + azure_content_safety_params: + endpoint: "" + key: "os.environ/AZURE_CONTENT_SAFETY_KEY" + thresholds: + Hate: 6 + SelfHarm: 8 + Sexual: 6 + Violence: 4 +``` + +:::info +`thresholds` are not required by default, but you can tune the values to your needs. +Default values is `4` for all categories +::: \ No newline at end of file diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index 2cea05c69a3..2735a883940 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -59,16 +59,6 @@ class _PROXY_AzureContentSafety( except: pass - def _severity(self, severity): - if severity >= 6: - return "high" - elif severity >= 4: - return "medium" - elif severity >= 2: - return "low" - else: - return "safe" - def _compute_result(self, response): result = {} @@ -80,7 +70,7 @@ class _PROXY_AzureContentSafety( if severity is not None: result[category] = { "filtered": severity >= self.thresholds[category], - "severity": self._severity(severity), + "severity": severity, } return result @@ -148,10 +138,10 @@ class _PROXY_AzureContentSafety( content=response.choices[0].message.content, source="output" ) - #async def async_post_call_streaming_hook( + # async def async_post_call_streaming_hook( # self, # user_api_key_dict: UserAPIKeyAuth, # response: str, - #): + # ): # self.print_verbose(f"Inside Azure Content-Safety Call-Stream Hook") # await self.test_violation(content=response, source="output") diff --git a/litellm/tests/test_azure_content_safety.py b/litellm/tests/test_azure_content_safety.py index f7d9d826868..3cc31003a7c 100644 --- a/litellm/tests/test_azure_content_safety.py +++ b/litellm/tests/test_azure_content_safety.py @@ -50,7 +50,7 @@ async def test_strict_input_filtering_01(): assert exc_info.value.detail["source"] == "input" assert exc_info.value.detail["category"] == "Hate" - assert exc_info.value.detail["severity"] == "low" + assert exc_info.value.detail["severity"] == 2 @pytest.mark.asyncio @@ -168,7 +168,7 @@ async def test_strict_output_filtering_01(): assert exc_info.value.detail["source"] == "output" assert exc_info.value.detail["category"] == "Hate" - assert exc_info.value.detail["severity"] == "low" + assert exc_info.value.detail["severity"] == 2 @pytest.mark.asyncio From ebbeb333c6820c100e9e6cf5dfd577adceda2714 Mon Sep 17 00:00:00 2001 From: Lunik Date: Sat, 4 May 2024 10:45:15 +0200 Subject: [PATCH 010/169] =?UTF-8?q?=E2=9C=8F=EF=B8=8F=20doc:=20typo=20in?= =?UTF-8?q?=20azure=20content=20safety?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- docs/my-website/docs/proxy/logging.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index f583765bb97..eaa65f09c76 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -3,7 +3,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# 🔎 Logging - Custom Callbacks, DataDog, Langfuse, s3 Bucket, Sentry, OpenTelemetry, Athina +# 🔎 Logging - Custom Callbacks, DataDog, Langfuse, s3 Bucket, Sentry, OpenTelemetry, Athina, Azure Content-Safety Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTelemetry, LangFuse, DynamoDB, s3 Bucket @@ -1019,7 +1019,7 @@ Deploy an Azure Content-Safety instance from the Azure Portal and get the `endpo **Step 1** Set Athina API key ```shell -AZURE_CONTENT_SAFETY_KEU = "" +AZURE_CONTENT_SAFETY_KEY = "" ``` **Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` From 8783fd4895107cd2aa9ac5e7d9feec27c3b79ed5 Mon Sep 17 00:00:00 2001 From: Lunik Date: Sat, 4 May 2024 10:45:39 +0200 Subject: [PATCH 011/169] =?UTF-8?q?=E2=9C=A8=20feat:=20Use=208=20severity?= =?UTF-8?q?=20levels=20for=20azure=20content=20safety?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- litellm/proxy/hooks/azure_content_safety.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index 2735a883940..fb9fc80ae01 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -15,8 +15,11 @@ class _PROXY_AzureContentSafety( try: from azure.ai.contentsafety.aio import ContentSafetyClient from azure.core.credentials import AzureKeyCredential - from azure.ai.contentsafety.models import TextCategory - from azure.ai.contentsafety.models import AnalyzeTextOptions + from azure.ai.contentsafety.models import ( + TextCategory, + AnalyzeTextOptions, + AnalyzeTextOutputType, + ) from azure.core.exceptions import HttpResponseError except Exception as e: raise Exception( @@ -26,6 +29,7 @@ class _PROXY_AzureContentSafety( self.api_key = api_key self.text_category = TextCategory self.analyze_text_options = AnalyzeTextOptions + self.analyze_text_output_type = AnalyzeTextOutputType self.azure_http_error = HttpResponseError self.thresholds = self._configure_thresholds(thresholds) @@ -79,7 +83,10 @@ class _PROXY_AzureContentSafety( self.print_verbose(f"Testing Azure Content-Safety for: {content}") # Construct a request - request = self.analyze_text_options(text=content) + request = self.analyze_text_options( + text=content, + output_type=self.analyze_text_output_type.EIGHT_SEVERITY_LEVELS, + ) # Analyze text try: From 1639a51f241d906d2e1ce0b4343bd581206f7751 Mon Sep 17 00:00:00 2001 From: Lunik Date: Sat, 4 May 2024 11:04:23 +0200 Subject: [PATCH 012/169] =?UTF-8?q?=F0=9F=94=8A=20fix:=20Correctly=20use?= =?UTF-8?q?=20verbose=20logging?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lunik --- litellm/proxy/hooks/azure_content_safety.py | 22 +++++++-------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index fb9fc80ae01..433571c15ce 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -55,14 +55,6 @@ class _PROXY_AzureContentSafety( return thresholds - def print_verbose(self, print_statement): - try: - verbose_proxy_logger.debug(print_statement) - if litellm.set_verbose: - print(print_statement) # noqa - except: - pass - def _compute_result(self, response): result = {} @@ -80,7 +72,7 @@ class _PROXY_AzureContentSafety( return result async def test_violation(self, content: str, source: str = None): - self.print_verbose(f"Testing Azure Content-Safety for: {content}") + verbose_proxy_logger.debug("Testing Azure Content-Safety for: %s", content) # Construct a request request = self.analyze_text_options( @@ -92,14 +84,14 @@ class _PROXY_AzureContentSafety( try: response = await self.client.analyze_text(request) except self.azure_http_error as e: - self.print_verbose( - f"Error in Azure Content-Safety: {traceback.format_exc()}" + verbose_proxy_logger.debug( + "Error in Azure Content-Safety: %s", traceback.format_exc() ) traceback.print_exc() raise result = self._compute_result(response) - self.print_verbose(f"Azure Content-Safety Result: {result}") + verbose_proxy_logger.debug("Azure Content-Safety Result: %s", result) for key, value in result.items(): if value["filtered"]: @@ -120,7 +112,7 @@ class _PROXY_AzureContentSafety( data: dict, call_type: str, # "completion", "embeddings", "image_generation", "moderation" ): - self.print_verbose(f"Inside Azure Content-Safety Pre-Call Hook") + verbose_proxy_logger.debug("Inside Azure Content-Safety Pre-Call Hook") try: if call_type == "completion" and "messages" in data: for m in data["messages"]: @@ -137,7 +129,7 @@ class _PROXY_AzureContentSafety( user_api_key_dict: UserAPIKeyAuth, response, ): - self.print_verbose(f"Inside Azure Content-Safety Post-Call Hook") + verbose_proxy_logger.debug("Inside Azure Content-Safety Post-Call Hook") if isinstance(response, litellm.ModelResponse) and isinstance( response.choices[0], litellm.utils.Choices ): @@ -150,5 +142,5 @@ class _PROXY_AzureContentSafety( # user_api_key_dict: UserAPIKeyAuth, # response: str, # ): - # self.print_verbose(f"Inside Azure Content-Safety Call-Stream Hook") + # verbose_proxy_logger.debug("Inside Azure Content-Safety Call-Stream Hook") # await self.test_violation(content=response, source="output") From 4b655d8b3343be0d5a537be59e58c12f29820c89 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 15:02:37 +0000 Subject: [PATCH 013/169] feat(util.py): Add OIDC support. --- litellm/tests/test_secret_manager.py | 33 +++++++++++++++++ litellm/utils.py | 54 ++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/litellm/tests/test_secret_manager.py b/litellm/tests/test_secret_manager.py index 3ea38f8061b..892a8831cfa 100644 --- a/litellm/tests/test_secret_manager.py +++ b/litellm/tests/test_secret_manager.py @@ -23,3 +23,36 @@ def test_aws_secret_manager(): print(f"secret_val: {secret_val}") assert secret_val == "sk-1234" + + +def redact_oidc_signature(secret_val): + # remove the last part of `.` and replace it with "SIGNATURE_REMOVED" + return secret_val.split(".")[:-1] + ["SIGNATURE_REMOVED"] + + +@pytest.mark.skipif(os.environ.get('K_SERVICE') is None, reason="Cannot run without being in GCP Cloud Run") +def test_oidc_google(): + secret_val = get_secret("oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + +@pytest.mark.skipif(os.environ.get('ACTIONS_ID_TOKEN_REQUEST_TOKEN') is None, reason="Cannot run without being in GitHub Actions") +def test_oidc_github(): + secret_val = get_secret("oidc/github/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + +@pytest.mark.skipif(os.environ.get('CIRCLE_OIDC_TOKEN') is None, reason="Cannot run without being in a CircleCI Runner") +def test_oidc_circleci(): + secret_val = get_secret("oidc/circleci/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") + + +@pytest.mark.skipif(os.environ.get('CIRCLE_OIDC_TOKEN_V2') is None, reason="Cannot run without being in a CircleCI Runner") +def test_oidc_circleci_v2(): + secret_val = get_secret("oidc/circleci_v2/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke") + + print(f"secret_val: {redact_oidc_signature(secret_val)}") diff --git a/litellm/utils.py b/litellm/utils.py index a938a0ba841..652406f64e8 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -33,6 +33,7 @@ from dataclasses import ( ) import litellm._service_logger # for storing API inputs, outputs, and metadata +from litellm.llms.custom_httpx.http_handler import HTTPHandler try: # this works in python 3.8 @@ -9288,6 +9289,59 @@ def get_secret( if secret_name.startswith("os.environ/"): secret_name = secret_name.replace("os.environ/", "") + # Example: oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/stability.stable-diffusion-xl-v1/invoke + if secret_name.startswith("oidc/"): + secret_name = secret_name.replace("oidc/", "") + oidc_provider, oidc_aud = secret_name.split("/", 1) + # TODO: Add caching for HTTP requests + match oidc_provider: + case "google": + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature + response = client.get( + "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity", + params={"audience": oidc_aud}, + headers={"Metadata-Flavor": "Google"}, + ) + if response.status_code == 200: + return response.text + else: + raise ValueError("Google OIDC provider failed") + case "circleci": + # https://circleci.com/docs/openid-connect-tokens/ + env_secret = os.getenv("CIRCLE_OIDC_TOKEN") + if env_secret is None: + raise ValueError("CIRCLE_OIDC_TOKEN not found in environment") + return env_secret + case "circleci_v2": + # https://circleci.com/docs/openid-connect-tokens/ + env_secret = os.getenv("CIRCLE_OIDC_TOKEN_V2") + if env_secret is None: + raise ValueError("CIRCLE_OIDC_TOKEN_V2 not found in environment") + return env_secret + case "github": + # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers#using-custom-actions + actions_id_token_request_url = os.getenv("ACTIONS_ID_TOKEN_REQUEST_URL") + actions_id_token_request_token = os.getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN") + if actions_id_token_request_url is None or actions_id_token_request_token is None: + raise ValueError("ACTIONS_ID_TOKEN_REQUEST_URL or ACTIONS_ID_TOKEN_REQUEST_TOKEN not found in environment") + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + response = client.get( + actions_id_token_request_url, + params={"audience": oidc_aud}, + headers={ + "Authorization": f"Bearer {actions_id_token_request_token}", + "Accept": "application/json; api-version=2.0", + }, + ) + if response.status_code == 200: + return response.text['value'] + else: + raise ValueError("Github OIDC provider failed") + case _: + raise ValueError("Unsupported OIDC provider") + + try: if litellm.secret_manager_client is not None: try: From 3ee0328b044e84e6bd7c3085681a468e69c3c646 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 15:01:46 +0000 Subject: [PATCH 014/169] feat(bedrock.py): Support using OIDC tokens. --- litellm/llms/bedrock.py | 42 +++++++++++++++++++++++- litellm/tests/test_bedrock_completion.py | 29 ++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/litellm/llms/bedrock.py b/litellm/llms/bedrock.py index 2f26ae4a9a0..fe0aca44e73 100644 --- a/litellm/llms/bedrock.py +++ b/litellm/llms/bedrock.py @@ -550,6 +550,7 @@ def init_bedrock_client( aws_session_name: Optional[str] = None, aws_profile_name: Optional[str] = None, aws_role_name: Optional[str] = None, + aws_web_identity_token: Optional[str] = None, extra_headers: Optional[dict] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, ): @@ -566,6 +567,7 @@ def init_bedrock_client( aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ] # Iterate over parameters and update if needed @@ -581,6 +583,7 @@ def init_bedrock_client( aws_session_name, aws_profile_name, aws_role_name, + aws_web_identity_token, ) = params_to_check ### SET REGION NAME @@ -619,7 +622,38 @@ def init_bedrock_client( config = boto3.session.Config() ### CHECK STS ### - if aws_role_name is not None and aws_session_name is not None: + if aws_web_identity_token is not None and aws_role_name is not None and aws_session_name is not None: + oidc_token = get_secret(aws_web_identity_token) + + if oidc_token is None: + raise BedrockError( + message="OIDC token could not be retrieved from secret manager.", + status_code=401, + ) + + sts_client = boto3.client( + "sts" + ) + + # https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/sts/client/assume_role_with_web_identity.html + sts_response = sts_client.assume_role_with_web_identity( + RoleArn=aws_role_name, + RoleSessionName=aws_session_name, + WebIdentityToken=oidc_token, + DurationSeconds=3600, + ) + + client = boto3.client( + service_name="bedrock-runtime", + aws_access_key_id=sts_response["Credentials"]["AccessKeyId"], + aws_secret_access_key=sts_response["Credentials"]["SecretAccessKey"], + aws_session_token=sts_response["Credentials"]["SessionToken"], + region_name=region_name, + endpoint_url=endpoint_url, + config=config, + ) + elif aws_role_name is not None and aws_session_name is not None: # use sts if role name passed in sts_client = boto3.client( "sts", @@ -752,6 +786,7 @@ def completion( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = optional_params.pop("aws_bedrock_client", None) @@ -766,6 +801,7 @@ def completion( aws_role_name=aws_role_name, aws_session_name=aws_session_name, aws_profile_name=aws_profile_name, + aws_web_identity_token=aws_web_identity_token, extra_headers=extra_headers, timeout=timeout, ) @@ -1288,6 +1324,7 @@ def embedding( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = init_bedrock_client( @@ -1295,6 +1332,7 @@ def embedding( aws_secret_access_key=aws_secret_access_key, aws_region_name=aws_region_name, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_web_identity_token=aws_web_identity_token, aws_role_name=aws_role_name, aws_session_name=aws_session_name, ) @@ -1377,6 +1415,7 @@ def image_generation( aws_bedrock_runtime_endpoint = optional_params.pop( "aws_bedrock_runtime_endpoint", None ) + aws_web_identity_token = optional_params.pop("aws_web_identity_token", None) # use passed in BedrockRuntime.Client if provided, otherwise create a new one client = init_bedrock_client( @@ -1384,6 +1423,7 @@ def image_generation( aws_secret_access_key=aws_secret_access_key, aws_region_name=aws_region_name, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_web_identity_token=aws_web_identity_token, aws_role_name=aws_role_name, aws_session_name=aws_session_name, timeout=timeout, diff --git a/litellm/tests/test_bedrock_completion.py b/litellm/tests/test_bedrock_completion.py index 3f5c831d73a..ef6774fd2f8 100644 --- a/litellm/tests/test_bedrock_completion.py +++ b/litellm/tests/test_bedrock_completion.py @@ -206,6 +206,35 @@ def test_completion_bedrock_claude_sts_client_auth(): # test_completion_bedrock_claude_sts_client_auth() +@pytest.mark.skipif(os.environ.get('CIRCLE_OIDC_TOKEN_V2') is None, reason="CIRCLE_OIDC_TOKEN_V2 is not set") +def test_completion_bedrock_claude_sts_oidc_auth(): + print("\ncalling bedrock claude with oidc auth") + import os + + aws_web_identity_token = "oidc/circleci_v2/" + aws_region_name = os.environ["AWS_REGION_NAME"] + aws_role_name = os.environ["AWS_TEMP_ROLE_NAME"] + + try: + litellm.set_verbose = True + + response = completion( + model="bedrock/anthropic.claude-instant-v1", + messages=messages, + max_tokens=10, + temperature=0.1, + aws_region_name=aws_region_name, + aws_web_identity_token=aws_web_identity_token, + aws_role_name=aws_role_name, + aws_session_name="my-test-session", + ) + # Add any assertions here to check the response + print(response) + except RateLimitError: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + def test_bedrock_extra_headers(): try: From d5767e940302f31defa9f640f59beacf060e666f Mon Sep 17 00:00:00 2001 From: Jean-Luc Duckworth Date: Tue, 7 May 2024 15:42:06 -0400 Subject: [PATCH 015/169] Expanding jwt access to other RS and PS algos. Updated to resolve merge conflicts. --- litellm/proxy/auth/handle_jwt.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 606ff682813..9c846e8f661 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -156,6 +156,11 @@ class JWTHandler: return public_key async def auth_jwt(self, token: str) -> dict: + # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html + # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret + # the key in different ways (e.g. HS* and RS*)." + algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512"], + audience = os.getenv("JWT_AUDIENCE") decode_options = None if audience is None: @@ -189,7 +194,7 @@ class JWTHandler: payload = jwt.decode( token, public_key_rsa, # type: ignore - algorithms=["RS256"], + algorithms=algorithms, options=decode_options, audience=audience, ) @@ -214,7 +219,7 @@ class JWTHandler: payload = jwt.decode( token, key, - algorithms=["RS256"], + algorithms=algorithms, audience=audience, options=decode_options ) From e268354acc3dbdbf0a2313aa7fd87fd485774b34 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 19:18:28 +0000 Subject: [PATCH 016/169] feat(azure.py): Support OIDC auth --- litellm/llms/azure.py | 66 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index e7af9d43b64..c2bbe54c152 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -8,6 +8,7 @@ from litellm.utils import ( CustomStreamWrapper, convert_to_model_response_object, TranscriptionResponse, + get_secret, ) from typing import Callable, Optional, BinaryIO from litellm import OpenAIConfig @@ -16,6 +17,7 @@ import httpx from .custom_httpx.azure_dall_e_2 import CustomHTTPTransport, AsyncCustomHTTPTransport from openai import AzureOpenAI, AsyncAzureOpenAI import uuid +import os class AzureOpenAIError(Exception): @@ -126,6 +128,51 @@ def select_azure_base_url_or_endpoint(azure_client_params: dict): return azure_client_params +def get_azure_ad_token_from_oidc(azure_ad_token: str): + azure_client_id = os.getenv("AZURE_CLIENT_ID", None) + azure_tenant = os.getenv("AZURE_TENANT_ID", None) + + if azure_client_id is None or azure_tenant is None: + raise AzureOpenAIError( + status_code=422, + message="AZURE_CLIENT_ID and AZURE_TENANT_ID must be set", + ) + + oidc_token = get_secret(azure_ad_token) + + if oidc_token is None: + raise AzureOpenAIError( + status_code=401, + message="OIDC token could not be retrieved from secret manager.", + ) + + req_token = httpx.get( + f"https://login.microsoftonline.com/{azure_tenant}/oauth2/v2.0/token", + data={ + "client_id": azure_client_id, + "grant_type": "client_credentials", + "scope": "https://cognitiveservices.azure.com/.default", + "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + "client_assertion": oidc_token, + }, + ) + + if req_token.status_code != 200: + raise AzureOpenAIError( + status_code=req_token.status_code, + message=req_token.text, + ) + + possible_azure_ad_token = req_token.json().get("access_token", None) + + if possible_azure_ad_token is None: + raise AzureOpenAIError( + status_code=422, message="Azure AD Token not returned" + ) + + return possible_azure_ad_token + + class AzureChatCompletion(BaseLLM): def __init__(self) -> None: super().__init__() @@ -137,6 +184,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: headers["api-key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) headers["Authorization"] = f"Bearer {azure_ad_token}" return headers @@ -189,6 +238,9 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) + azure_client_params["azure_ad_token"] = azure_ad_token if acompletion is True: @@ -276,6 +328,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AzureOpenAI(**azure_client_params) @@ -351,6 +405,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token # setting Azure client @@ -422,6 +478,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AzureOpenAI(**azure_client_params) @@ -478,6 +536,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if client is None: azure_client = AsyncAzureOpenAI(**azure_client_params) @@ -599,6 +659,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token ## LOGGING @@ -755,6 +817,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if aimg_generation == True: @@ -833,6 +897,8 @@ class AzureChatCompletion(BaseLLM): if api_key is not None: azure_client_params["api_key"] = api_key elif azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) azure_client_params["azure_ad_token"] = azure_ad_token if max_retries is not None: From 9a0bb36865a3ed10325eab67f5d74169fd152891 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 19:43:18 +0000 Subject: [PATCH 017/169] fix+feat(router.py): Fix missing azure_ad_token, and allow use OIDC auth --- litellm/router.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 4353da804b4..24a926e5d9c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -45,6 +45,7 @@ from litellm.types.router import ( RetryPolicy, ) from litellm.integrations.custom_logger import CustomLogger +from litellm.llms.azure import get_azure_ad_token_from_oidc class Router: @@ -2089,6 +2090,10 @@ class Router: raise ValueError( f"api_base is required for Azure OpenAI. Set it on your config. Model - {model}" ) + azure_ad_token = litellm_params.get("azure_ad_token") + if azure_ad_token is not None: + if azure_ad_token.startswith("oidc/"): + azure_ad_token = get_azure_ad_token_from_oidc(azure_ad_token) if api_version is None: api_version = "2023-07-01-preview" if "gateway.ai.cloudflare.com" in api_base: @@ -2099,6 +2104,7 @@ class Router: cache_key = f"{model_id}_async_client" _client = openai.AsyncAzureOpenAI( api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=timeout, @@ -2123,6 +2129,7 @@ class Router: cache_key = f"{model_id}_client" _client = openai.AzureOpenAI( # type: ignore api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=timeout, @@ -2147,6 +2154,7 @@ class Router: cache_key = f"{model_id}_stream_async_client" _client = openai.AsyncAzureOpenAI( # type: ignore api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=stream_timeout, @@ -2171,6 +2179,7 @@ class Router: cache_key = f"{model_id}_stream_client" _client = openai.AzureOpenAI( # type: ignore api_key=api_key, + azure_ad_token=azure_ad_token, base_url=api_base, api_version=api_version, timeout=stream_timeout, From cb49fb004d7bba866d229760931256bf96971f8e Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 19:51:57 +0000 Subject: [PATCH 018/169] fix(azure.py): Correct invalid .get to a .post for OIDC --- litellm/llms/azure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/azure.py b/litellm/llms/azure.py index c2bbe54c152..1e807b5e7dc 100644 --- a/litellm/llms/azure.py +++ b/litellm/llms/azure.py @@ -146,7 +146,7 @@ def get_azure_ad_token_from_oidc(azure_ad_token: str): message="OIDC token could not be retrieved from secret manager.", ) - req_token = httpx.get( + req_token = httpx.post( f"https://login.microsoftonline.com/{azure_tenant}/oauth2/v2.0/token", data={ "client_id": azure_client_id, From d60aa8282eab4f1a80e900aafe21e13cc0ab17d7 Mon Sep 17 00:00:00 2001 From: Jean-Luc Duckworth Date: Tue, 7 May 2024 16:08:36 -0400 Subject: [PATCH 019/169] Fixed typo. test_jwt.py tests pass --- litellm/proxy/auth/handle_jwt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/auth/handle_jwt.py b/litellm/proxy/auth/handle_jwt.py index 9c846e8f661..18c0d7b2cec 100644 --- a/litellm/proxy/auth/handle_jwt.py +++ b/litellm/proxy/auth/handle_jwt.py @@ -159,7 +159,7 @@ class JWTHandler: # Supported algos: https://pyjwt.readthedocs.io/en/stable/algorithms.html # "Warning: Make sure not to mix symmetric and asymmetric algorithms that interpret # the key in different ways (e.g. HS* and RS*)." - algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512"], + algorithms = ["RS256", "RS384", "RS512", "PS256", "PS384", "PS512"] audience = os.getenv("JWT_AUDIENCE") decode_options = None From 44b1b219115f9d4804ca9dec2b82441059a3c128 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Tue, 7 May 2024 21:20:15 +0000 Subject: [PATCH 020/169] feat(utils.py) - Add OIDC caching for Google Cloud Run and GitHub Actions. --- litellm/utils.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 652406f64e8..f5d3b974b6d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -34,6 +34,8 @@ from dataclasses import ( import litellm._service_logger # for storing API inputs, outputs, and metadata from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.caching import DualCache +oidc_cache = DualCache() try: # this works in python 3.8 @@ -9291,11 +9293,15 @@ def get_secret( # Example: oidc/google/https://bedrock-runtime.us-east-1.amazonaws.com/model/stability.stable-diffusion-xl-v1/invoke if secret_name.startswith("oidc/"): - secret_name = secret_name.replace("oidc/", "") - oidc_provider, oidc_aud = secret_name.split("/", 1) + secret_name_split = secret_name.replace("oidc/", "") + oidc_provider, oidc_aud = secret_name_split.split("/", 1) # TODO: Add caching for HTTP requests match oidc_provider: case "google": + oidc_token = oidc_cache.get_cache(key=secret_name) + if oidc_token is not None: + return oidc_token + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature response = client.get( @@ -9304,7 +9310,9 @@ def get_secret( headers={"Metadata-Flavor": "Google"}, ) if response.status_code == 200: - return response.text + oidc_token = response.text + oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=3600 - 60) + return oidc_token else: raise ValueError("Google OIDC provider failed") case "circleci": @@ -9325,6 +9333,11 @@ def get_secret( actions_id_token_request_token = os.getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN") if actions_id_token_request_url is None or actions_id_token_request_token is None: raise ValueError("ACTIONS_ID_TOKEN_REQUEST_URL or ACTIONS_ID_TOKEN_REQUEST_TOKEN not found in environment") + + oidc_token = oidc_cache.get_cache(key=secret_name) + if oidc_token is not None: + return oidc_token + client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) response = client.get( actions_id_token_request_url, @@ -9335,7 +9348,9 @@ def get_secret( }, ) if response.status_code == 200: - return response.text['value'] + oidc_token = response.text['value'] + oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=300 - 5) + return oidc_token else: raise ValueError("Github OIDC provider failed") case _: From 77b3acb396269ac31dc1916c17ff2b9e430d1585 Mon Sep 17 00:00:00 2001 From: David Manouchehri Date: Wed, 8 May 2024 14:34:45 +0000 Subject: [PATCH 021/169] fix(router.py): Add missing azure_ad_token param. --- litellm/router.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/router.py b/litellm/router.py index 24a926e5d9c..ea8d17286ff 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2212,6 +2212,7 @@ class Router: "api_key": api_key, "azure_endpoint": api_base, "api_version": api_version, + "azure_ad_token": azure_ad_token, } from litellm.llms.azure import select_azure_base_url_or_endpoint From fc51a3631e235c11fbbcee4fbb0e013c8b41f69a Mon Sep 17 00:00:00 2001 From: Merlinvt Date: Thu, 9 May 2024 15:16:34 +0200 Subject: [PATCH 022/169] add additional models from openrouter --- model_prices_and_context_window.json | 138 ++++++++++++++++++++++++++- 1 file changed, 137 insertions(+), 1 deletion(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 10c70a858da..dbd812038f4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1571,6 +1571,142 @@ "litellm_provider": "replicate", "mode": "chat" }, + "openrouter/mistralai/mixtral-8x22b-instruct": { + "max_tokens": 65536, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000065, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/microsoft/wizardlm-2-8x22b:nitro": { + "max_tokens": 65536, + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000001, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/google/gemini-pro-1.5": { + "max_tokens": 8192, + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 0.0000025, + "output_cost_per_token": 0.0000075, + "input_cost_per_image": 0.00265, + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/mistralai/mixtral-8x22b-instruct": { + "max_tokens": 65536, + "input_cost_per_token": 0.00000065, + "output_cost_per_token": 0.00000065, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/cohere/command-r-plus": { + "max_tokens": 128000, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/databricks/dbrx-instruct": { + "max_tokens": 32768, + "input_cost_per_token": 0.0000006, + "output_cost_per_token": 0.0000006, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/anthropic/claude-3-haiku": { + "max_tokens": 200000, + "input_cost_per_token": 0.00000025, + "output_cost_per_token": 0.00000125, + "input_cost_per_image": 0.0004, // Calculated for per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/anthropic/claude-3-sonnet": { + "max_tokens": 200000, + "input_cost_per_token": 0.000003, + "output_cost_per_token": 0.000015, + "input_cost_per_image": 0.0048, // Per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/mistralai/mistral-large": { + "max_tokens": 32000, + "input_cost_per_token": 0.000008, + "output_cost_per_token": 0.000024, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/cognitivecomputations/dolphin-mixtral-8x7b": { + "max_tokens": 32769, + "input_cost_per_token": 0.0000005, + "output_cost_per_token": 0.0000005, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/google/gemini-pro-vision": { + "max_tokens": 45875, + "input_cost_per_token": 0.000000125, + "output_cost_per_token": 0.000000375, + "input_cost_per_image": 0.0025, // Per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, + "openrouter/fireworks/firellava-13b": { + "max_tokens": 4096, + "input_cost_per_token": 0.0000002, + "output_cost_per_token": 0.0000002, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-8b-instruct:free": { + "max_tokens": 8200, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-8b-instruct:extended": { + "max_tokens": 16000, + "input_cost_per_token": 0.000000225, + "output_cost_per_token": 0.00000225, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-70b-instruct:nitro": { + "max_tokens": 8200, + "input_cost_per_token": 0.0000009, + "output_cost_per_token": 0.0000009, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/meta-llama/llama-3-70b-instruct": { + "max_tokens": 8200, + "input_cost_per_token": 0.00000059, + "output_cost_per_token": 0.00000079, + "litellm_provider": "openrouter", + "mode": "chat" + }, + "openrouter/openai/gpt-4-vision-preview": { + "max_tokens": 130000, + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00003, + "input_cost_per_image": 0.01445, // Per 1000 images. + "litellm_provider": "openrouter", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true + }, "openrouter/openai/gpt-3.5-turbo": { "max_tokens": 4095, "input_cost_per_token": 0.0000015, @@ -3226,4 +3362,4 @@ "mode": "embedding" } -} +} \ No newline at end of file From ccdd2046af00e420983ee655d56aa84d38315ce4 Mon Sep 17 00:00:00 2001 From: Merlinvt Date: Thu, 9 May 2024 15:20:32 +0200 Subject: [PATCH 023/169] fixes --- model_prices_and_context_window.json | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dbd812038f4..2f575757e59 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1571,13 +1571,6 @@ "litellm_provider": "replicate", "mode": "chat" }, - "openrouter/mistralai/mixtral-8x22b-instruct": { - "max_tokens": 65536, - "input_cost_per_token": 0.00000065, - "output_cost_per_token": 0.00000065, - "litellm_provider": "openrouter", - "mode": "chat" - }, "openrouter/microsoft/wizardlm-2-8x22b:nitro": { "max_tokens": 65536, "input_cost_per_token": 0.000001, @@ -1622,7 +1615,7 @@ "max_tokens": 200000, "input_cost_per_token": 0.00000025, "output_cost_per_token": 0.00000125, - "input_cost_per_image": 0.0004, // Calculated for per 1000 images. + "input_cost_per_image": 0.0004, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1632,7 +1625,7 @@ "max_tokens": 200000, "input_cost_per_token": 0.000003, "output_cost_per_token": 0.000015, - "input_cost_per_image": 0.0048, // Per 1000 images. + "input_cost_per_image": 0.0048, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1656,7 +1649,7 @@ "max_tokens": 45875, "input_cost_per_token": 0.000000125, "output_cost_per_token": 0.000000375, - "input_cost_per_image": 0.0025, // Per 1000 images. + "input_cost_per_image": 0.0025, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1701,7 +1694,7 @@ "max_tokens": 130000, "input_cost_per_token": 0.00001, "output_cost_per_token": 0.00003, - "input_cost_per_image": 0.01445, // Per 1000 images. + "input_cost_per_image": 0.01445, "litellm_provider": "openrouter", "mode": "chat", "supports_function_calling": true, @@ -1847,13 +1840,6 @@ "litellm_provider": "openrouter", "mode": "chat" }, - "openrouter/meta-llama/llama-3-70b-instruct": { - "max_tokens": 8192, - "input_cost_per_token": 0.0000008, - "output_cost_per_token": 0.0000008, - "litellm_provider": "openrouter", - "mode": "chat" - }, "j2-ultra": { "max_tokens": 8192, "max_input_tokens": 8192, From 265d777894a16a4daad03b18a91341229ecae788 Mon Sep 17 00:00:00 2001 From: Merlinvt Date: Thu, 9 May 2024 15:27:14 +0200 Subject: [PATCH 024/169] fixes 2 --- model_prices_and_context_window.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 2f575757e59..1ade08fe35e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1663,28 +1663,28 @@ "mode": "chat" }, "openrouter/meta-llama/llama-3-8b-instruct:free": { - "max_tokens": 8200, + "max_tokens": 8192, "input_cost_per_token": 0.0, "output_cost_per_token": 0.0, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/meta-llama/llama-3-8b-instruct:extended": { - "max_tokens": 16000, + "max_tokens": 16384, "input_cost_per_token": 0.000000225, "output_cost_per_token": 0.00000225, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/meta-llama/llama-3-70b-instruct:nitro": { - "max_tokens": 8200, + "max_tokens": 8192, "input_cost_per_token": 0.0000009, "output_cost_per_token": 0.0000009, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/meta-llama/llama-3-70b-instruct": { - "max_tokens": 8200, + "max_tokens": 8192, "input_cost_per_token": 0.00000059, "output_cost_per_token": 0.00000079, "litellm_provider": "openrouter", @@ -1750,14 +1750,14 @@ "tool_use_system_prompt_tokens": 395 }, "openrouter/google/palm-2-chat-bison": { - "max_tokens": 8000, + "max_tokens": 25804, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", "mode": "chat" }, "openrouter/google/palm-2-codechat-bison": { - "max_tokens": 8000, + "max_tokens": 20070, "input_cost_per_token": 0.0000005, "output_cost_per_token": 0.0000005, "litellm_provider": "openrouter", From 4d5b4a5293ff56473f44c856acaa98af171abd62 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 08:35:35 -0700 Subject: [PATCH 025/169] add stream_options to text_completion --- litellm/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index 186b87060b5..60e71a412e6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3195,6 +3195,7 @@ def text_completion( Union[str, List[str]] ] = None, # Optional: Sequences where the API will stop generating further tokens. stream: Optional[bool] = None, # Optional: Whether to stream back partial progress. + stream_options: Optional[dict] = None, suffix: Optional[ str ] = None, # Optional: The suffix that comes after a completion of inserted text. @@ -3272,6 +3273,8 @@ def text_completion( optional_params["stop"] = stop if stream is not None: optional_params["stream"] = stream + if stream_options is not None: + optional_params["stream_options"] = stream_options if suffix is not None: optional_params["suffix"] = suffix if temperature is not None: @@ -3382,7 +3385,9 @@ def text_completion( if kwargs.get("acompletion", False) == True: return response if stream == True or kwargs.get("stream", False) == True: - response = TextCompletionStreamWrapper(completion_stream=response, model=model) + response = TextCompletionStreamWrapper( + completion_stream=response, model=model, stream_options=stream_options + ) return response transformed_logprobs = None # only supported for TGI models From 66053f14ae9d67ceb49b96055b0f8dd8831f9f9a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 08:37:40 -0700 Subject: [PATCH 026/169] stream_options for text-completionopenai --- litellm/llms/openai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index d542cbe0795..674cc86a256 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -1205,6 +1205,7 @@ class OpenAITextCompletion(BaseLLM): model=model, custom_llm_provider="text-completion-openai", logging_obj=logging_obj, + stream_options=data.get("stream_options", None), ) for chunk in streamwrapper: @@ -1243,6 +1244,7 @@ class OpenAITextCompletion(BaseLLM): model=model, custom_llm_provider="text-completion-openai", logging_obj=logging_obj, + stream_options=data.get("stream_options", None), ) async for transformed_chunk in streamwrapper: From a29fcc057b5f00a7eb662a12e4e60d9560aba2a4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 08:41:31 -0700 Subject: [PATCH 027/169] test - stream_options on OpenAI text_completion --- litellm/tests/test_streaming.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index 7d639d7a3c4..93d7567eb90 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -1534,6 +1534,39 @@ def test_openai_stream_options_call(): assert all(chunk.usage is None for chunk in chunks[:-1]) +def test_openai_stream_options_call_text_completion(): + litellm.set_verbose = False + response = litellm.text_completion( + model="gpt-3.5-turbo-instruct", + prompt="say GM - we're going to make it ", + stream=True, + stream_options={"include_usage": True}, + max_tokens=10, + ) + usage = None + chunks = [] + for chunk in response: + print("chunk: ", chunk) + chunks.append(chunk) + + last_chunk = chunks[-1] + print("last chunk: ", last_chunk) + + """ + Assert that: + - Last Chunk includes Usage + - All chunks prior to last chunk have usage=None + """ + + assert last_chunk.usage is not None + assert last_chunk.usage.total_tokens > 0 + assert last_chunk.usage.prompt_tokens > 0 + assert last_chunk.usage.completion_tokens > 0 + + # assert all non last chunks have usage=None + assert all(chunk.usage is None for chunk in chunks[:-1]) + + def test_openai_text_completion_call(): try: litellm.set_verbose = True From e0b1eff1eb67d57a56be0bfa87cf7b636c7c57a4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 08:42:25 -0700 Subject: [PATCH 028/169] feat - support stream_options for text completion --- litellm/utils.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 64a644f159f..1932b8af6a9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10026,16 +10026,19 @@ class CustomStreamWrapper: text = "" is_finished = False finish_reason = None + usage = None choices = getattr(chunk, "choices", []) if len(choices) > 0: text = choices[0].text if choices[0].finish_reason is not None: is_finished = True finish_reason = choices[0].finish_reason + usage = getattr(chunk, "usage", None) return { "text": text, "is_finished": is_finished, "finish_reason": finish_reason, + "usage": usage, } except Exception as e: @@ -10565,6 +10568,11 @@ class CustomStreamWrapper: print_verbose(f"completion obj content: {completion_obj['content']}") if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] + if ( + self.stream_options + and self.stream_options.get("include_usage", False) == True + ): + model_response.usage = response_obj["usage"] elif self.custom_llm_provider == "azure_text": response_obj = self.handle_azure_text_completion_chunk(chunk) completion_obj["content"] = response_obj["text"] @@ -11094,9 +11102,10 @@ class CustomStreamWrapper: class TextCompletionStreamWrapper: - def __init__(self, completion_stream, model): + def __init__(self, completion_stream, model, stream_options): self.completion_stream = completion_stream self.model = model + self.stream_options = stream_options def __iter__(self): return self @@ -11120,6 +11129,14 @@ class TextCompletionStreamWrapper: text_choices["index"] = chunk["choices"][0]["index"] text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] response["choices"] = [text_choices] + + # only pass usage when stream_options["include_usage"] is True + if ( + self.stream_options + and self.stream_options.get("include_usage", False) == True + ): + response["usage"] = chunk.get("usage", None) + return response except Exception as e: raise Exception( From 6634ea37e9f8b9a20d2fef7cbc3ae0a7b55d438f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 09:54:44 -0700 Subject: [PATCH 029/169] fix TextCompletionStreamWrapper --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 1932b8af6a9..09f1bb2e2b3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -11102,7 +11102,7 @@ class CustomStreamWrapper: class TextCompletionStreamWrapper: - def __init__(self, completion_stream, model, stream_options): + def __init__(self, completion_stream, model, stream_options: Optional[dict] = None): self.completion_stream = completion_stream self.model = model self.stream_options = stream_options From 84b2af8e6c91719c36ebf2601af6193b75e51a68 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 12:27:44 -0700 Subject: [PATCH 030/169] fix show docker run on repo --- .github/workflows/interpret_load_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py index 9d95c768fcd..de52f47ad8c 100644 --- a/.github/workflows/interpret_load_test.py +++ b/.github/workflows/interpret_load_test.py @@ -64,6 +64,11 @@ if __name__ == "__main__": ) # Replace with your repository's username and name latest_release = repo.get_latest_release() print("got latest release: ", latest_release) + print(latest_release.title) + print(latest_release.tag_name) + + release_version = latest_release.title + print("latest release body: ", latest_release.body) print("markdown table: ", markdown_table) @@ -74,6 +79,18 @@ if __name__ == "__main__": start_index = latest_release.body.find("Load Test LiteLLM Proxy Results") existing_release_body = latest_release.body[:start_index] + docker_run_command = f""" + ## Docker Run LiteLLM Proxy + + ``` + docker run \\ + -e STORE_MODEL_IN_DB=True \\ + -p 4000:4000 \\ + ghcr.io/berriai/litellm:main-{release_version} + ``` + """ + print("docker run command: ", docker_run_command) + new_release_body = ( existing_release_body + "\n\n" From 30b7e9f776475b2d84df9e5deb12d6ebaa44f349 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 12:28:14 -0700 Subject: [PATCH 031/169] temp fix laod test --- .github/workflows/load_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index ddf613fa660..9af700d244a 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -30,7 +30,7 @@ jobs: URL: "https://litellm-database-docker-build-production.up.railway.app/" USERS: "100" RATE: "10" - RUNTIME: "300s" + RUNTIME: "3s" - name: Process Load Test Stats run: | echo "Current working directory: $PWD" From 50b4167a27822069a22c94d28e1b73da1b5bf44a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 12:30:35 -0700 Subject: [PATCH 032/169] fix interpret load test --- .github/workflows/interpret_load_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py index de52f47ad8c..b61f1ce6721 100644 --- a/.github/workflows/interpret_load_test.py +++ b/.github/workflows/interpret_load_test.py @@ -80,6 +80,7 @@ if __name__ == "__main__": existing_release_body = latest_release.body[:start_index] docker_run_command = f""" + \n\n ## Docker Run LiteLLM Proxy ``` @@ -93,6 +94,7 @@ if __name__ == "__main__": new_release_body = ( existing_release_body + + docker_run_command + "\n\n" + "### Don't want to maintain your internal proxy? get in touch 🎉" + "\nHosted Proxy Alpha: https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat" From c8662234c7b6c3d26b696d9284ec83dea1dfbe87 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 12:38:38 -0700 Subject: [PATCH 033/169] fix docker run command on release notes --- .github/workflows/interpret_load_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/interpret_load_test.py b/.github/workflows/interpret_load_test.py index b61f1ce6721..b1a28e069b8 100644 --- a/.github/workflows/interpret_load_test.py +++ b/.github/workflows/interpret_load_test.py @@ -80,15 +80,15 @@ if __name__ == "__main__": existing_release_body = latest_release.body[:start_index] docker_run_command = f""" - \n\n - ## Docker Run LiteLLM Proxy +\n\n +## Docker Run LiteLLM Proxy - ``` - docker run \\ - -e STORE_MODEL_IN_DB=True \\ - -p 4000:4000 \\ - ghcr.io/berriai/litellm:main-{release_version} - ``` +``` +docker run \\ +-e STORE_MODEL_IN_DB=True \\ +-p 4000:4000 \\ +ghcr.io/berriai/litellm:main-{release_version} +``` """ print("docker run command: ", docker_run_command) From bf99311f5c3dcae825dee6763af90f40a9b18c24 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 9 May 2024 12:50:24 -0700 Subject: [PATCH 034/169] fix load test length --- .github/workflows/load_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml index 9af700d244a..ddf613fa660 100644 --- a/.github/workflows/load_test.yml +++ b/.github/workflows/load_test.yml @@ -30,7 +30,7 @@ jobs: URL: "https://litellm-database-docker-build-production.up.railway.app/" USERS: "100" RATE: "10" - RUNTIME: "3s" + RUNTIME: "300s" - name: Process Load Test Stats run: | echo "Current working directory: $PWD" From e3f25a4a1fbaa5de962845286863009e1ab00ced Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 13:05:41 -0700 Subject: [PATCH 035/169] fix(auth_checks.py): fix 'get_end_user_object' await cache get --- litellm/proxy/auth/auth_checks.py | 6 ++---- litellm/proxy/proxy_server.py | 7 +++++-- litellm/tests/test_key_generate_prisma.py | 9 ++++++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 920de3cc822..62e5eba0132 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -206,11 +206,9 @@ async def get_end_user_object( if end_user_id is None: return None - + _key = "end_user_id:{}".format(end_user_id) # check if in cache - cached_user_obj = user_api_key_cache.async_get_cache( - key="end_user_id:{}".format(end_user_id) - ) + cached_user_obj = await user_api_key_cache.async_get_cache(key=_key) if cached_user_obj is not None: if isinstance(cached_user_obj, dict): return LiteLLM_EndUserTable(**cached_user_obj) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index f50fddf7fbc..1ba29dcc699 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1834,6 +1834,9 @@ async def update_cache( ) async def _update_end_user_cache(): + if end_user_id is None or response_cost is None: + return + _id = "end_user_id:{}".format(end_user_id) try: # Fetch the existing cost for the given user @@ -1846,7 +1849,7 @@ async def update_cache( if litellm.max_end_user_budget is not None: max_end_user_budget = litellm.max_end_user_budget existing_spend_obj = LiteLLM_EndUserTable( - user_id=_id, + user_id=end_user_id, spend=0, blocked=False, litellm_budget_table=LiteLLM_BudgetTable( @@ -1874,7 +1877,7 @@ async def update_cache( existing_spend_obj.spend = new_spend user_api_key_cache.set_cache(key=_id, value=existing_spend_obj.json()) except Exception as e: - verbose_proxy_logger.debug( + verbose_proxy_logger.error( f"An error occurred updating end user cache: {str(e)}\n\n{traceback.format_exc()}" ) diff --git a/litellm/tests/test_key_generate_prisma.py b/litellm/tests/test_key_generate_prisma.py index 08618c98891..e6f2437e7d7 100644 --- a/litellm/tests/test_key_generate_prisma.py +++ b/litellm/tests/test_key_generate_prisma.py @@ -418,9 +418,16 @@ def test_call_with_user_over_budget(prisma_client): print(vars(e)) +def test_end_user_cache_write_unit_test(): + """ + assert end user object is being written to cache as expected + """ + pass + + def test_call_with_end_user_over_budget(prisma_client): # Test if a user passed to /chat/completions is tracked & fails when they cross their budget - # we only check this when litellm.max_user_budget is set + # we only check this when litellm.max_end_user_budget is set import random setattr(litellm.proxy.proxy_server, "prisma_client", prisma_client) From 927d36148f5c82ba3f85cd79f81cf4c211bd6d7b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 13:21:00 -0700 Subject: [PATCH 036/169] feat(proxy_server.py): expose new `/team/list` endpoint Closes https://github.com/BerriAI/litellm/issues/3523 --- litellm/proxy/proxy_server.py | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1ba29dcc699..0456881cd55 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7313,6 +7313,43 @@ async def unblock_team( return record +@router.get( + "/team/list", tags=["team management"], dependencies=[Depends(user_api_key_auth)] +) +async def list_team( + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + [Admin-only] List all available teams + + ``` + curl --location --request GET 'http://0.0.0.0:4000/team/list' \ + --header 'Authorization: Bearer sk-1234' + ``` + """ + global prisma_client + + if user_api_key_dict.user_role != "proxy_admin": + raise HTTPException( + status_code=401, + detail={ + "error": "Admin-only endpoint. Your user role={}".format( + user_api_key_dict.user_role + ) + }, + ) + + if prisma_client is None: + raise HTTPException( + status_code=400, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + + response = await prisma_client.db.litellm_teamtable.find_many() + + return response + + #### ORGANIZATION MANAGEMENT #### From c4295e16672ca612b2bc665e5fef71953de09e51 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 13:54:24 -0700 Subject: [PATCH 037/169] test(test_least_busy_routing.py): avoid deployments with low rate limits --- litellm/tests/test_least_busy_routing.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/litellm/tests/test_least_busy_routing.py b/litellm/tests/test_least_busy_routing.py index cb9d59e7557..203a03c63c4 100644 --- a/litellm/tests/test_least_busy_routing.py +++ b/litellm/tests/test_least_busy_routing.py @@ -150,9 +150,9 @@ async def test_router_atext_completion_streaming(): { "model_name": "azure-model", "litellm_params": { - "model": "azure/gpt-35-turbo", - "api_key": "os.environ/AZURE_EUROPE_API_KEY", - "api_base": "https://my-endpoint-europe-berri-992.openai.azure.com", + "model": "azure/gpt-turbo", + "api_key": "os.environ/AZURE_FRANCE_API_KEY", + "api_base": "https://openai-france-1234.openai.azure.com", "rpm": 6, }, "model_info": {"id": 2}, @@ -160,9 +160,9 @@ async def test_router_atext_completion_streaming(): { "model_name": "azure-model", "litellm_params": { - "model": "azure/gpt-35-turbo", - "api_key": "os.environ/AZURE_CANADA_API_KEY", - "api_base": "https://my-endpoint-canada-berri992.openai.azure.com", + "model": "azure/gpt-turbo", + "api_key": "os.environ/AZURE_FRANCE_API_KEY", + "api_base": "https://openai-france-1234.openai.azure.com", "rpm": 6, }, "model_info": {"id": 3}, @@ -193,7 +193,7 @@ async def test_router_atext_completion_streaming(): ## check if calls equally distributed cache_dict = router.cache.get_cache(key=cache_key) for k, v in cache_dict.items(): - assert v == 1 + assert v == 1, f"Failed. K={k} called v={v} times, cache_dict={cache_dict}" # asyncio.run(test_router_atext_completion_streaming()) From acb615957dad014c795cf360d58fbe3a9f4fa096 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 13:58:45 -0700 Subject: [PATCH 038/169] fix(utils.py): change error log to be debug --- litellm/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 20716f43d00..6da29603893 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5938,7 +5938,7 @@ def get_api_base(model: str, optional_params: dict) -> Optional[str]: model=model, **optional_params ) # convert to pydantic object except Exception as e: - verbose_logger.error("Error occurred in getting api base - {}".format(str(e))) + verbose_logger.debug("Error occurred in getting api base - {}".format(str(e))) return None # get llm provider From 5c6a382d3b50566e38769521405e89d7d8439d97 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 15:41:33 -0700 Subject: [PATCH 039/169] refactor(main.py): trigger new build --- litellm/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/main.py b/litellm/main.py index 99e556bfa25..5ab3fd7c452 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -14,6 +14,7 @@ import dotenv, traceback, random, asyncio, time, contextvars from copy import deepcopy import httpx import litellm + from ._logging import verbose_logger from litellm import ( # type: ignore client, From 43b2050cc25006f607307ce3efb4b0089ab98f8d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 15:41:40 -0700 Subject: [PATCH 040/169] =?UTF-8?q?bump:=20version=201.36.4=20=E2=86=92=20?= =?UTF-8?q?1.37.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 399b188c26a..a9854cf692a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.36.4" +version = "1.37.0" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.36.4" +version = "1.37.0" version_files = [ "pyproject.toml:^version" ] From c42f1ce2c65375bde058b1f2b399bbd88ea09426 Mon Sep 17 00:00:00 2001 From: Nick Wong Date: Thu, 9 May 2024 16:13:26 -0700 Subject: [PATCH 041/169] removed extra default dict return, which causes error if user_role is a string --- litellm/proxy/proxy_server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 4bb8dee7f9c..86c22186dbf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -1065,9 +1065,7 @@ async def user_api_key_auth( user_id_information, list ): _user = user_id_information[0] - user_role = _user.get("user_role", {}).get( - "user_role", "unknown" - ) + user_role = _user.get("user_role", "unknown") user_id = _user.get("user_id", "unknown") raise Exception( f"Only proxy admin can be used to generate, delete, update info for new keys/users/teams. Route={route}. Your role={user_role}. Your user_id={user_id}" From 186c0ec77bec2900a184d8628ff1457dd25b114e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 16:39:43 -0700 Subject: [PATCH 042/169] feat(predibase.py): add support for predibase provider Closes https://github.com/BerriAI/litellm/issues/1253 --- litellm/__init__.py | 4 + litellm/llms/huggingface_restapi.py | 2 +- litellm/llms/predibase.py | 417 ++++++++++++++++++++++++++++ litellm/main.py | 54 ++++ litellm/tests/test_completion.py | 23 ++ litellm/utils.py | 2 +- 6 files changed, 500 insertions(+), 2 deletions(-) create mode 100644 litellm/llms/predibase.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 4f72504f692..ccf3657fe02 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -71,9 +71,11 @@ maritalk_key: Optional[str] = None ai21_key: Optional[str] = None ollama_key: Optional[str] = None openrouter_key: Optional[str] = None +predibase_key: Optional[str] = None huggingface_key: Optional[str] = None vertex_project: Optional[str] = None vertex_location: Optional[str] = None +predibase_tenant_id: Optional[str] = None togetherai_api_key: Optional[str] = None cloudflare_api_key: Optional[str] = None baseten_key: Optional[str] = None @@ -532,6 +534,7 @@ provider_list: List = [ "xinference", "fireworks_ai", "watsonx", + "predibase", "custom", # custom apis ] @@ -644,6 +647,7 @@ from .utils import ( ) from .llms.huggingface_restapi import HuggingfaceConfig from .llms.anthropic import AnthropicConfig +from .llms.predibase import PredibaseConfig from .llms.anthropic_text import AnthropicTextConfig from .llms.replicate import ReplicateConfig from .llms.cohere import CohereConfig diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 29377328908..b250f301383 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -322,9 +322,9 @@ class Huggingface(BaseLLM): encoding, api_key, logging_obj, + optional_params: dict, custom_prompt_dict={}, acompletion: bool = False, - optional_params=None, litellm_params=None, logger_fn=None, ): diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py new file mode 100644 index 00000000000..728a98b04b5 --- /dev/null +++ b/litellm/llms/predibase.py @@ -0,0 +1,417 @@ +# What is this? +## Controller file for Predibase Integration - https://predibase.com/ + + +import os, types +import json +from enum import Enum +import requests, copy # type: ignore +import time +from typing import Callable, Optional, List, Literal, Union +from litellm.utils import ( + ModelResponse, + Usage, + map_finish_reason, + CustomStreamWrapper, + Message, + Choices, +) +import litellm +from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from .base import BaseLLM +import httpx # type: ignore + + +class PredibaseError(Exception): + def __init__( + self, + status_code, + message, + request: Optional[httpx.Request] = None, + response: Optional[httpx.Response] = None, + ): + self.status_code = status_code + self.message = message + if request is not None: + self.request = request + else: + self.request = httpx.Request( + method="POST", + url="https://docs.predibase.com/user-guide/inference/rest_api", + ) + if response is not None: + self.response = response + else: + self.response = httpx.Response( + status_code=status_code, request=self.request + ) + super().__init__( + self.message + ) # Call the base class constructor with the parameters it needs + + +class PredibaseConfig: + """ + Reference: https://docs.predibase.com/user-guide/inference/rest_api + + """ + + adapter_id: Optional[str] = None + adapter_source: Optional[Literal["pbase", "hub", "s3"]] = None + best_of: Optional[int] = None + decoder_input_details: bool = True # on by default - get the finish reason + details: Optional[bool] = True # enables returning logprobs + best of + max_new_tokens: int = ( + 256 # openai default - requests hang if max_new_tokens not given + ) + repetition_penalty: Optional[float] = None + return_full_text: Optional[bool] = ( + False # by default don't return the input as part of the output + ) + seed: Optional[int] = None + stop: Optional[List[str]] = None + temperature: Optional[float] = None + top_k: Optional[int] = None + top_p: Optional[int] = None + truncate: Optional[int] = None + typical_p: Optional[float] = None + watermark: Optional[bool] = None + + def __init__( + self, + best_of: Optional[int] = None, + decoder_input_details: Optional[bool] = None, + details: Optional[bool] = None, + max_new_tokens: Optional[int] = None, + repetition_penalty: Optional[float] = None, + return_full_text: Optional[bool] = None, + seed: Optional[int] = None, + stop: Optional[List[str]] = None, + temperature: Optional[float] = None, + top_k: Optional[int] = None, + top_p: Optional[int] = None, + truncate: Optional[int] = None, + typical_p: Optional[float] = None, + watermark: Optional[bool] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self): + return ["stream", "temperature", "max_tokens", "top_p", "stop", "n"] + + +class PredibaseChatCompletion(BaseLLM): + def __init__(self) -> None: + super().__init__() + + def validate_environment(self, api_key: Optional[str], user_headers: dict) -> dict: + if api_key is None: + raise ValueError( + "Missing Predibase API Key - A call is being made to predibase but no key is set either in the environment variables or via params" + ) + headers = { + "content-type": "application/json", + "Authorization": "Bearer {}".format(api_key), + } + if user_headers is not None and isinstance(user_headers, dict): + headers = {**headers, **user_headers} + return headers + + def output_parser(self, generated_text: str): + """ + Parse the output text to remove any special characters. In our current approach we just check for ChatML tokens. + + Initial issue that prompted this - https://github.com/BerriAI/litellm/issues/763 + """ + chat_template_tokens = [ + "<|assistant|>", + "<|system|>", + "<|user|>", + "", + "", + ] + for token in chat_template_tokens: + if generated_text.strip().startswith(token): + generated_text = generated_text.replace(token, "", 1) + if generated_text.endswith(token): + generated_text = generated_text[::-1].replace(token[::-1], "", 1)[::-1] + return generated_text + + def process_response( + self, + model: str, + response: requests.Response, + model_response: ModelResponse, + stream: bool, + logging_obj: litellm.utils.Logging, + optional_params: dict, + api_key: str, + data: dict, + messages: list, + print_verbose, + encoding, + ) -> ModelResponse: + ## LOGGING + logging_obj.post_call( + input=messages, + api_key=api_key, + original_response=response.text, + additional_args={"complete_input_dict": data}, + ) + print_verbose(f"raw model_response: {response.text}") + ## RESPONSE OBJECT + try: + completion_response = response.json() + except: + raise PredibaseError( + message=response.text, status_code=response.status_code + ) + if "error" in completion_response: + raise PredibaseError( + message=str(completion_response["error"]), + status_code=response.status_code, + ) + else: + if ( + not isinstance(completion_response, dict) + or "generated_text" not in completion_response + ): + raise PredibaseError( + status_code=422, + message=f"response is not in expected format - {completion_response}", + ) + + if len(completion_response["generated_text"]) > 0: + model_response["choices"][0]["message"]["content"] = self.output_parser( + completion_response["generated_text"] + ) + ## GETTING LOGPROBS + FINISH REASON + if ( + "details" in completion_response + and "tokens" in completion_response["details"] + ): + model_response.choices[0].finish_reason = completion_response[ + "details" + ]["finish_reason"] + sum_logprob = 0 + for token in completion_response[0]["details"]["tokens"]: + if token["logprob"] != None: + sum_logprob += token["logprob"] + model_response["choices"][0][ + "message" + ]._logprob = ( + sum_logprob # [TODO] move this to using the actual logprobs + ) + if "best_of" in optional_params and optional_params["best_of"] > 1: + if ( + "details" in completion_response[0] + and "best_of_sequences" in completion_response[0]["details"] + ): + choices_list = [] + for idx, item in enumerate( + completion_response[0]["details"]["best_of_sequences"] + ): + sum_logprob = 0 + for token in item["tokens"]: + if token["logprob"] != None: + sum_logprob += token["logprob"] + if len(item["generated_text"]) > 0: + message_obj = Message( + content=self.output_parser(item["generated_text"]), + logprobs=sum_logprob, + ) + else: + message_obj = Message(content=None) + choice_obj = Choices( + finish_reason=item["finish_reason"], + index=idx + 1, + message=message_obj, + ) + choices_list.append(choice_obj) + model_response["choices"].extend(choices_list) + + ## CALCULATING USAGE + prompt_tokens = 0 + try: + prompt_tokens = len( + encoding.encode(model_response["choices"][0]["message"]["content"]) + ) ##[TODO] use a model-specific tokenizer here + except: + # this should remain non blocking we should not block a response returning if calculating usage fails + pass + output_text = model_response["choices"][0]["message"].get("content", "") + if output_text is not None and len(output_text) > 0: + completion_tokens = 0 + try: + completion_tokens = len( + encoding.encode( + model_response["choices"][0]["message"].get("content", "") + ) + ) ##[TODO] use a model-specific tokenizer + except: + # this should remain non blocking we should not block a response returning if calculating usage fails + pass + else: + completion_tokens = 0 + + total_tokens = prompt_tokens + completion_tokens + + model_response["created"] = int(time.time()) + model_response["model"] = model + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + model_response.usage = usage # type: ignore + return model_response + + def completion( + self, + model: str, + messages: list, + api_base: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key: str, + logging_obj, + optional_params: dict, + tenant_id: str, + acompletion=None, + litellm_params=None, + logger_fn=None, + headers: dict = {}, + ): + headers = self.validate_environment(api_key, headers) + completion_url = "" + input_text = "" + base_url = "https://serving.app.predibase.com" + if "https" in model: + completion_url = model + elif api_base: + base_url = api_base + elif "PREDIBASE_API_BASE" in os.environ: + base_url = os.getenv("PREDIBASE_API_BASE", "") + + completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}/generate" + + if model in custom_prompt_dict: + # check if the model has a registered custom prompt + model_prompt_details = custom_prompt_dict[model] + prompt = custom_prompt( + role_dict=model_prompt_details["roles"], + initial_prompt_value=model_prompt_details["initial_prompt_value"], + final_prompt_value=model_prompt_details["final_prompt_value"], + messages=messages, + ) + else: + prompt = prompt_factory(model=model, messages=messages) + + ## Load Config + config = litellm.PredibaseConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + data = { + "inputs": prompt, + "parameters": optional_params, + } + if optional_params.get("stream") and optional_params["stream"] == True: + data["stream"] = True + input_text = prompt + ## LOGGING + logging_obj.pre_call( + input=input_text, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "headers": headers, + "api_base": completion_url, + "acompletion": acompletion, + }, + ) + ## COMPLETION CALL + if acompletion is True: + ### ASYNC STREAMING + if optional_params.get("stream", False): + return self.async_streaming(logging_obj=logging_obj, api_base=completion_url, data=data, headers=headers, model_response=model_response, model=model, timeout=timeout) # type: ignore + else: + ### ASYNC COMPLETION + return self.acompletion(api_base=completion_url, data=data, headers=headers, model_response=model_response, task=task, encoding=encoding, input_text=input_text, model=model, optional_params=optional_params, timeout=timeout) # type: ignore + + ### SYNC STREAMING + if "stream" in optional_params and optional_params["stream"] == True: + response = requests.post( + completion_url, + headers=headers, + data=json.dumps(data), + stream=optional_params["stream"], + ) + return response.iter_lines() + ### SYNC COMPLETION + else: + payload = json.dumps( + { + "inputs": "What is your name?", + "parameters": {"max_new_tokens": 20, "temperature": 0.1}, + } + # data + ) + response = requests.post( + url="https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate", + headers=headers, + data=payload, + ) + + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=optional_params.get("stream", False), + logging_obj=logging_obj, # type: ignore + optional_params=optional_params, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + encoding=encoding, + ) + + async def async_completion(self): + pass + + async def async_streaming(self): + pass + + def streaming(self): + pass + + def embedding(self, *args, **kwargs): + pass diff --git a/litellm/main.py b/litellm/main.py index 5ab3fd7c452..f634fd16d94 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -74,6 +74,7 @@ from .llms.azure_text import AzureTextCompletion from .llms.anthropic import AnthropicChatCompletion from .llms.anthropic_text import AnthropicTextCompletion from .llms.huggingface_restapi import Huggingface +from .llms.predibase import PredibaseChatCompletion from .llms.prompt_templates.factory import ( prompt_factory, custom_prompt, @@ -110,6 +111,7 @@ anthropic_text_completions = AnthropicTextCompletion() azure_chat_completions = AzureChatCompletion() azure_text_completions = AzureTextCompletion() huggingface = Huggingface() +predibase_chat_completions = PredibaseChatCompletion() ####### COMPLETION ENDPOINTS ################ @@ -1785,6 +1787,58 @@ def completion( ) return response response = model_response + elif custom_llm_provider == "predibase": + tenant_id = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) + + api_base = ( + optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.predibase_key + or get_secret("PREDIBASE_API_KEY") + ) + + model_response = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion == False + ): + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="predibase", + logging_obj=logging, + ) + return response + response = model_response elif custom_llm_provider == "ai21": custom_llm_provider = "ai21" ai21_key = ( diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 32b65faea5f..7f0977b155c 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -85,6 +85,29 @@ def test_completion_azure_command_r(): pytest.fail(f"Error occurred: {e}") +@pytest.mark.skip(reason="local test") +def test_completion_predibase(): + try: + litellm.set_verbose = True + + response = completion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + ) + + print(response) + except litellm.Timeout as e: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + + +# test_completion_predibase() + + def test_completion_claude(): litellm.set_verbose = True litellm.cache = None diff --git a/litellm/utils.py b/litellm/utils.py index 6da29603893..7ccb5e8ff9f 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -369,7 +369,7 @@ class ChatCompletionMessageToolCall(OpenAIObject): class Message(OpenAIObject): def __init__( self, - content="default", + content: Optional[str] = "default", role="assistant", logprobs=None, function_call=None, From d7189c21fdd57e8cf445c7ef82c9d73df3dfaf7e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 17:41:27 -0700 Subject: [PATCH 043/169] feat(predibase.py): support async_completion + streaming (sync + async) finishes up pr --- litellm/llms/predibase.py | 166 +- litellm/main.py | 7 +- litellm/tests/log.txt | 6981 +++++++++++++++++++++++++++++- litellm/tests/test_completion.py | 33 +- litellm/tests/test_streaming.py | 67 +- litellm/utils.py | 50 + 6 files changed, 7196 insertions(+), 108 deletions(-) diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index 728a98b04b5..f3935984d0d 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -60,8 +60,8 @@ class PredibaseConfig: adapter_id: Optional[str] = None adapter_source: Optional[Literal["pbase", "hub", "s3"]] = None best_of: Optional[int] = None - decoder_input_details: bool = True # on by default - get the finish reason - details: Optional[bool] = True # enables returning logprobs + best of + decoder_input_details: Optional[bool] = None + details: bool = True # enables returning logprobs + best of max_new_tokens: int = ( 256 # openai default - requests hang if max_new_tokens not given ) @@ -124,6 +124,9 @@ class PredibaseConfig: class PredibaseChatCompletion(BaseLLM): def __init__(self) -> None: + self.async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=litellm.request_timeout, connect=5.0) + ) super().__init__() def validate_environment(self, api_key: Optional[str], user_headers: dict) -> dict: @@ -162,7 +165,7 @@ class PredibaseChatCompletion(BaseLLM): def process_response( self, model: str, - response: requests.Response, + response: Union[requests.Response, httpx.Response], model_response: ModelResponse, stream: bool, logging_obj: litellm.utils.Logging, @@ -216,7 +219,7 @@ class PredibaseChatCompletion(BaseLLM): "details" ]["finish_reason"] sum_logprob = 0 - for token in completion_response[0]["details"]["tokens"]: + for token in completion_response["details"]["tokens"]: if token["logprob"] != None: sum_logprob += token["logprob"] model_response["choices"][0][ @@ -226,12 +229,12 @@ class PredibaseChatCompletion(BaseLLM): ) if "best_of" in optional_params and optional_params["best_of"] > 1: if ( - "details" in completion_response[0] - and "best_of_sequences" in completion_response[0]["details"] + "details" in completion_response + and "best_of_sequences" in completion_response["details"] ): choices_list = [] for idx, item in enumerate( - completion_response[0]["details"]["best_of_sequences"] + completion_response["details"]["best_of_sequences"] ): sum_logprob = 0 for token in item["tokens"]: @@ -305,7 +308,7 @@ class PredibaseChatCompletion(BaseLLM): litellm_params=None, logger_fn=None, headers: dict = {}, - ): + ) -> Union[ModelResponse, CustomStreamWrapper]: headers = self.validate_environment(api_key, headers) completion_url = "" input_text = "" @@ -317,7 +320,12 @@ class PredibaseChatCompletion(BaseLLM): elif "PREDIBASE_API_BASE" in os.environ: base_url = os.getenv("PREDIBASE_API_BASE", "") - completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}/generate" + completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}" + + if optional_params.get("stream", False) == True: + completion_url += "/generate_stream" + else: + completion_url += "/generate" if model in custom_prompt_dict: # check if the model has a registered custom prompt @@ -339,12 +347,12 @@ class PredibaseChatCompletion(BaseLLM): ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in optional_params[k] = v + stream = optional_params.pop("stream", False) + data = { "inputs": prompt, "parameters": optional_params, } - if optional_params.get("stream") and optional_params["stream"] == True: - data["stream"] = True input_text = prompt ## LOGGING logging_obj.pre_call( @@ -360,34 +368,62 @@ class PredibaseChatCompletion(BaseLLM): ## COMPLETION CALL if acompletion is True: ### ASYNC STREAMING - if optional_params.get("stream", False): - return self.async_streaming(logging_obj=logging_obj, api_base=completion_url, data=data, headers=headers, model_response=model_response, model=model, timeout=timeout) # type: ignore + if stream == True: + return self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=completion_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) # type: ignore else: ### ASYNC COMPLETION - return self.acompletion(api_base=completion_url, data=data, headers=headers, model_response=model_response, task=task, encoding=encoding, input_text=input_text, model=model, optional_params=optional_params, timeout=timeout) # type: ignore + return self.async_completion( + model=model, + messages=messages, + data=data, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + stream=False, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) # type: ignore ### SYNC STREAMING - if "stream" in optional_params and optional_params["stream"] == True: + if stream == True: response = requests.post( completion_url, headers=headers, data=json.dumps(data), - stream=optional_params["stream"], + stream=stream, ) - return response.iter_lines() + response = CustomStreamWrapper( + response.iter_lines(), + model, + custom_llm_provider="predibase", + logging_obj=logging_obj, + ) + return response ### SYNC COMPLETION else: - payload = json.dumps( - { - "inputs": "What is your name?", - "parameters": {"max_new_tokens": 20, "temperature": 0.1}, - } - # data - ) response = requests.post( - url="https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate", + url=completion_url, headers=headers, - data=payload, + data=json.dumps(data), ) return self.process_response( @@ -404,14 +440,80 @@ class PredibaseChatCompletion(BaseLLM): encoding=encoding, ) - async def async_completion(self): - pass + async def async_completion( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + stream, + data: dict, + optional_params: dict, + litellm_params=None, + logger_fn=None, + headers={}, + ) -> ModelResponse: - async def async_streaming(self): - pass + response = await self.async_handler.post( + api_base, headers=headers, data=json.dumps(data) + ) + return self.process_response( + model=model, + response=response, + model_response=model_response, + stream=stream, + logging_obj=logging_obj, + api_key=api_key, + data=data, + messages=messages, + print_verbose=print_verbose, + optional_params=optional_params, + encoding=encoding, + ) - def streaming(self): - pass + async def async_streaming( + self, + model: str, + messages: list, + api_base: str, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key, + logging_obj, + data: dict, + optional_params=None, + litellm_params=None, + logger_fn=None, + headers={}, + ) -> CustomStreamWrapper: + + data["stream"] = True + response = await self.async_handler.post( + url="https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate_stream", + headers=headers, + data=json.dumps(data), + stream=True, + ) + + if response.status_code != 200: + raise PredibaseError( + status_code=response.status_code, message=response.text + ) + + completion_stream = response.aiter_lines() + + streamwrapper = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider="predibase", + logging_obj=logging_obj, + ) + return streamwrapper def embedding(self, *args, **kwargs): pass diff --git a/litellm/main.py b/litellm/main.py index f634fd16d94..cec49d35f7e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -320,6 +320,7 @@ async def acompletion( or custom_llm_provider == "gemini" or custom_llm_provider == "sagemaker" or custom_llm_provider == "anthropic" + or custom_llm_provider == "predibase" or custom_llm_provider in litellm.openai_compatible_providers ): # currently implemented aiohttp calls for just azure, openai, hf, ollama, vertex ai soon all. init_response = await loop.run_in_executor(None, func_with_context) @@ -1831,12 +1832,6 @@ def completion( and optional_params["stream"] == True and acompletion == False ): - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="predibase", - logging_obj=logging, - ) return response response = model_response elif custom_llm_provider == "ai21": diff --git a/litellm/tests/log.txt b/litellm/tests/log.txt index 2c76ac9478c..4d3027355c3 100644 --- a/litellm/tests/log.txt +++ b/litellm/tests/log.txt @@ -1,25 +1,6884 @@ ============================= test session starts ============================== -platform darwin -- Python 3.11.6, pytest-7.3.1, pluggy-1.3.0 +platform darwin -- Python 3.11.9, pytest-7.3.1, pluggy-1.3.0 rootdir: /Users/krrishdholakia/Documents/litellm/litellm/tests plugins: timeout-2.2.0, asyncio-0.23.2, anyio-3.7.1, xdist-3.3.1 asyncio: mode=Mode.STRICT -collected 1 item +collected 2 items -test_completion.py Chunks sorted -token_counter messages received: [{'role': 'user', 'content': 'what is the capital of congo?'}] -Token Counter - using generic token counter, for model=gemini-1.5-pro-latest -LiteLLM: Utils - Counting tokens for OpenAI model=gpt-3.5-turbo -.Token Counter - using generic token counter, for model=gemini-1.5-pro-latest -LiteLLM: Utils - Counting tokens for OpenAI model=gpt-3.5-turbo -Looking up model=gemini/gemini-1.5-pro-latest in model_cost_map -Success: model=gemini/gemini-1.5-pro-latest in model_cost_map -prompt_tokens=15; completion_tokens=1 -Returned custom cost for model=gemini/gemini-1.5-pro-latest - prompt_tokens_cost_usd_dollar: 0, completion_tokens_cost_usd_dollar: 0 -final cost: 0; prompt_tokens_cost_usd_dollar: 0; completion_tokens_cost_usd_dollar: 0 - [100%] +test_streaming.py .Token Counter - using hugging face token counter, for model=llama-3-8b-instruct +Looking up model=llama-3-8b-instruct in model_cost_map +F [100%] +=================================== FAILURES =================================== +__________________ test_completion_predibase_streaming[True] ___________________ + +model = 'llama-3-8b-instruct' +messages = [{'content': 'What is the meaning of life?', 'role': 'user'}] +timeout = 600.0, temperature = None, top_p = None, n = None, stream = True +stream_options = None, stop = None, max_tokens = None, presence_penalty = None +frequency_penalty = None, logit_bias = None, user = None, response_format = None +seed = None, tools = None, tool_choice = None, logprobs = None +top_logprobs = None, deployment_id = None, extra_headers = None +functions = None, function_call = None, base_url = None, api_version = None +api_key = 'pb_Qg9YbQo7UqqHdu0ozxN_aw', model_list = None +kwargs = {'api_base': 'https://serving.app.predibase.com', 'litellm_call_id': 'cf0ea464-1b45-4473-8e55-6bf6809df7a7', 'litellm_logging_obj': , 'tenant_id': 'c4768f95'} +args = {'acompletion': False, 'api_base': None, 'api_key': 'pb_Qg9YbQo7UqqHdu0ozxN_aw', 'api_version': None, ...} +api_base = None, mock_response = None, force_timeout = 600, logger_fn = None +verbose = False, custom_llm_provider = 'predibase' + + @client + def completion( + model: str, + # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create + messages: List = [], + timeout: Optional[Union[float, str, httpx.Timeout]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + n: Optional[int] = None, + stream: Optional[bool] = None, + stream_options: Optional[dict] = None, + stop=None, + max_tokens: Optional[int] = None, + presence_penalty: Optional[float] = None, + frequency_penalty: Optional[float] = None, + logit_bias: Optional[dict] = None, + user: Optional[str] = None, + # openai v1.0+ new params + response_format: Optional[dict] = None, + seed: Optional[int] = None, + tools: Optional[List] = None, + tool_choice: Optional[str] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + deployment_id=None, + extra_headers: Optional[dict] = None, + # soon to be deprecated params by OpenAI + functions: Optional[List] = None, + function_call: Optional[str] = None, + # set api_base, api_version, api_key + base_url: Optional[str] = None, + api_version: Optional[str] = None, + api_key: Optional[str] = None, + model_list: Optional[list] = None, # pass in a list of api_base,keys, etc. + # Optional liteLLM function params + **kwargs, + ) -> Union[ModelResponse, CustomStreamWrapper]: + """ + Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly) + Parameters: + model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/ + messages (List): A list of message objects representing the conversation context (default is an empty list). + + OPTIONAL PARAMS + functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list). + function_call (str, optional): The name of the function to call within the conversation (default is an empty string). + temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0). + top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0). + n (int, optional): The number of completions to generate (default is 1). + stream (bool, optional): If True, return a streaming response (default is False). + stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true. + stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens. + max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity). + presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far. + frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far. + logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion. + user (str, optional): A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse. + logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message + top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used. + metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc. + api_base (str, optional): Base URL for the API (default is None). + api_version (str, optional): API version (default is None). + api_key (str, optional): API key (default is None). + model_list (list, optional): List of api base, version, keys + extra_headers (dict, optional): Additional headers to include in the request. + + LITELLM Specific Params + mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None). + custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock" + max_retries (int, optional): The number of retries to attempt (default is 0). + Returns: + ModelResponse: A response object containing the generated completion and associated metadata. + + Note: + - This function is used to perform completions() using the specified language model. + - It supports various optional parameters for customizing the completion behavior. + - If 'mock_response' is provided, a mock completion response is returned for testing or debugging. + """ + ######### unpacking kwargs ##################### + args = locals() + api_base = kwargs.get("api_base", None) + mock_response = kwargs.get("mock_response", None) + force_timeout = kwargs.get("force_timeout", 600) ## deprecated + logger_fn = kwargs.get("logger_fn", None) + verbose = kwargs.get("verbose", False) + custom_llm_provider = kwargs.get("custom_llm_provider", None) + litellm_logging_obj = kwargs.get("litellm_logging_obj", None) + id = kwargs.get("id", None) + metadata = kwargs.get("metadata", None) + model_info = kwargs.get("model_info", None) + proxy_server_request = kwargs.get("proxy_server_request", None) + fallbacks = kwargs.get("fallbacks", None) + headers = kwargs.get("headers", None) + num_retries = kwargs.get("num_retries", None) ## deprecated + max_retries = kwargs.get("max_retries", None) + context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None) + organization = kwargs.get("organization", None) + ### CUSTOM MODEL COST ### + input_cost_per_token = kwargs.get("input_cost_per_token", None) + output_cost_per_token = kwargs.get("output_cost_per_token", None) + input_cost_per_second = kwargs.get("input_cost_per_second", None) + output_cost_per_second = kwargs.get("output_cost_per_second", None) + ### CUSTOM PROMPT TEMPLATE ### + initial_prompt_value = kwargs.get("initial_prompt_value", None) + roles = kwargs.get("roles", None) + final_prompt_value = kwargs.get("final_prompt_value", None) + bos_token = kwargs.get("bos_token", None) + eos_token = kwargs.get("eos_token", None) + preset_cache_key = kwargs.get("preset_cache_key", None) + hf_model_name = kwargs.get("hf_model_name", None) + supports_system_message = kwargs.get("supports_system_message", None) + ### TEXT COMPLETION CALLS ### + text_completion = kwargs.get("text_completion", False) + atext_completion = kwargs.get("atext_completion", False) + ### ASYNC CALLS ### + acompletion = kwargs.get("acompletion", False) + client = kwargs.get("client", None) + ### Admin Controls ### + no_log = kwargs.get("no-log", False) + ######## end of unpacking kwargs ########### + openai_params = [ + "functions", + "function_call", + "temperature", + "temperature", + "top_p", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "request_timeout", + "api_base", + "api_version", + "api_key", + "deployment_id", + "organization", + "base_url", + "default_headers", + "timeout", + "response_format", + "seed", + "tools", + "tool_choice", + "max_retries", + "logprobs", + "top_logprobs", + "extra_headers", + ] + litellm_params = [ + "metadata", + "acompletion", + "atext_completion", + "text_completion", + "caching", + "mock_response", + "api_key", + "api_version", + "api_base", + "force_timeout", + "logger_fn", + "verbose", + "custom_llm_provider", + "litellm_logging_obj", + "litellm_call_id", + "use_client", + "id", + "fallbacks", + "azure", + "headers", + "model_list", + "num_retries", + "context_window_fallback_dict", + "retry_policy", + "roles", + "final_prompt_value", + "bos_token", + "eos_token", + "request_timeout", + "complete_response", + "self", + "client", + "rpm", + "tpm", + "max_parallel_requests", + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_second", + "output_cost_per_second", + "hf_model_name", + "model_info", + "proxy_server_request", + "preset_cache_key", + "caching_groups", + "ttl", + "cache", + "no-log", + "base_model", + "stream_timeout", + "supports_system_message", + "region_name", + "allowed_model_region", + ] + default_params = openai_params + litellm_params + non_default_params = { + k: v for k, v in kwargs.items() if k not in default_params + } # model-specific params - pass them straight to the model/provider + + ### TIMEOUT LOGIC ### + timeout = timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + + try: + if base_url is not None: + api_base = base_url + if max_retries is not None: # openai allows openai.OpenAI(max_retries=3) + num_retries = max_retries + logging = litellm_logging_obj + fallbacks = fallbacks or litellm.model_fallbacks + if fallbacks is not None: + return completion_with_fallbacks(**args) + if model_list is not None: + deployments = [ + m["litellm_params"] for m in model_list if m["model_name"] == model + ] + return batch_completion_models(deployments=deployments, **args) + if litellm.model_alias_map and model in litellm.model_alias_map: + model = litellm.model_alias_map[ + model + ] # update the model to the actual value if an alias has been passed in + model_response = ModelResponse() + setattr(model_response, "usage", litellm.Usage()) + if ( + kwargs.get("azure", False) == True + ): # don't remove flag check, to remain backwards compatible for repos like Codium + custom_llm_provider = "azure" + if deployment_id != None: # azure llms + model = deployment_id + custom_llm_provider = "azure" + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + if model_response is not None and hasattr(model_response, "_hidden_params"): + model_response._hidden_params["custom_llm_provider"] = custom_llm_provider + model_response._hidden_params["region_name"] = kwargs.get( + "aws_region_name", None + ) # support region-based pricing for bedrock + + ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### + if input_cost_per_token is not None and output_cost_per_token is not None: + print_verbose(f"Registering model={model} in model cost map") + litellm.register_model( + { + f"{custom_llm_provider}/{model}": { + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + "litellm_provider": custom_llm_provider, + }, + model: { + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + "litellm_provider": custom_llm_provider, + }, + } + ) + elif ( + input_cost_per_second is not None + ): # time based pricing just needs cost in place + output_cost_per_second = output_cost_per_second + litellm.register_model( + { + f"{custom_llm_provider}/{model}": { + "input_cost_per_second": input_cost_per_second, + "output_cost_per_second": output_cost_per_second, + "litellm_provider": custom_llm_provider, + }, + model: { + "input_cost_per_second": input_cost_per_second, + "output_cost_per_second": output_cost_per_second, + "litellm_provider": custom_llm_provider, + }, + } + ) + ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### + custom_prompt_dict = {} # type: ignore + if ( + initial_prompt_value + or roles + or final_prompt_value + or bos_token + or eos_token + ): + custom_prompt_dict = {model: {}} + if initial_prompt_value: + custom_prompt_dict[model]["initial_prompt_value"] = initial_prompt_value + if roles: + custom_prompt_dict[model]["roles"] = roles + if final_prompt_value: + custom_prompt_dict[model]["final_prompt_value"] = final_prompt_value + if bos_token: + custom_prompt_dict[model]["bos_token"] = bos_token + if eos_token: + custom_prompt_dict[model]["eos_token"] = eos_token + + if ( + supports_system_message is not None + and isinstance(supports_system_message, bool) + and supports_system_message == False + ): + messages = map_system_message_pt(messages=messages) + model_api_key = get_api_key( + llm_provider=custom_llm_provider, dynamic_api_key=api_key + ) # get the api key from the environment if required for the model + + if dynamic_api_key is not None: + api_key = dynamic_api_key + # check if user passed in any of the OpenAI optional params + optional_params = get_optional_params( + functions=functions, + function_call=function_call, + temperature=temperature, + top_p=top_p, + n=n, + stream=stream, + stream_options=stream_options, + stop=stop, + max_tokens=max_tokens, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + logit_bias=logit_bias, + user=user, + # params to identify the model + model=model, + custom_llm_provider=custom_llm_provider, + response_format=response_format, + seed=seed, + tools=tools, + tool_choice=tool_choice, + max_retries=max_retries, + logprobs=logprobs, + top_logprobs=top_logprobs, + extra_headers=extra_headers, + **non_default_params, + ) + + if litellm.add_function_to_prompt and optional_params.get( + "functions_unsupported_model", None + ): # if user opts to add it to prompt, when API doesn't support function calling + functions_unsupported_model = optional_params.pop( + "functions_unsupported_model" + ) + messages = function_call_prompt( + messages=messages, functions=functions_unsupported_model + ) + + # For logging - save the values of the litellm-specific params passed in + litellm_params = get_litellm_params( + acompletion=acompletion, + api_key=api_key, + force_timeout=force_timeout, + logger_fn=logger_fn, + verbose=verbose, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + litellm_call_id=kwargs.get("litellm_call_id", None), + model_alias_map=litellm.model_alias_map, + completion_call_id=id, + metadata=metadata, + model_info=model_info, + proxy_server_request=proxy_server_request, + preset_cache_key=preset_cache_key, + no_log=no_log, + ) + logging.update_environment_variables( + model=model, + user=user, + optional_params=optional_params, + litellm_params=litellm_params, + ) + if mock_response: + return mock_completion( + model, + messages, + stream=stream, + mock_response=mock_response, + logging=logging, + acompletion=acompletion, + ) + if custom_llm_provider == "azure": + # azure configs + api_type = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret("AZURE_AD_TOKEN") + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + elif custom_llm_provider == "azure_text": + # azure configs + api_type = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret("AZURE_AD_TOKEN") + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_text_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + elif ( + model in litellm.open_ai_chat_completion_models + or custom_llm_provider == "custom_openai" + or custom_llm_provider == "deepinfra" + or custom_llm_provider == "perplexity" + or custom_llm_provider == "groq" + or custom_llm_provider == "deepseek" + or custom_llm_provider == "anyscale" + or custom_llm_provider == "mistral" + or custom_llm_provider == "openai" + or custom_llm_provider == "together_ai" + or custom_llm_provider in litellm.openai_compatible_providers + or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo + ): # allow user to make an openai call with a custom base + # note: if a user sets a custom base - we should ensure this works + # allow for the setting of dynamic and stateful api-bases + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + openai.organization = ( + organization + or litellm.organization + or get_secret("OPENAI_ORGANIZATION") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.OpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + try: + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + organization=organization, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + elif ( + custom_llm_provider == "text-completion-openai" + or "ft:babbage-002" in model + or "ft:davinci-002" in model # support for finetuned completion models + ): + openai.api_type = "openai" + + api_base = ( + api_base + or litellm.api_base + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + openai.api_version = None + # set API KEY + + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.OpenAITextCompletionConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + if litellm.organization: + openai.organization = litellm.organization + + if ( + len(messages) > 0 + and "content" in messages[0] + and type(messages[0]["content"]) == list + ): + # text-davinci-003 can accept a string or array, if it's an array, assume the array is set in messages[0]['content'] + # https://platform.openai.com/docs/api-reference/completions/create + prompt = messages[0]["content"] + else: + prompt = " ".join([message["content"] for message in messages]) # type: ignore + + ## COMPLETION CALL + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + client=client, # pass AsyncOpenAI, OpenAI client + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) + + if ( + optional_params.get("stream", False) == False + and acompletion == False + and text_completion == False + ): + # convert to chat completion response + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + response = _response + elif ( + "replicate" in model + or custom_llm_provider == "replicate" + or model in litellm.replicate_models + ): + # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") + replicate_key = None + replicate_key = ( + api_key + or litellm.replicate_key + or litellm.api_key + or get_secret("REPLICATE_API_KEY") + or get_secret("REPLICATE_API_TOKEN") + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("REPLICATE_API_BASE") + or "https://api.replicate.com/v1" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = replicate.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=replicate_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + ) + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + model_response = CustomStreamWrapper(model_response, model, logging_obj=logging, custom_llm_provider="replicate") # type: ignore + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=replicate_key, + original_response=model_response, + ) + + response = model_response + + elif custom_llm_provider == "anthropic": + api_key = ( + api_key + or litellm.anthropic_key + or litellm.api_key + or os.environ.get("ANTHROPIC_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + if (model == "claude-2") or (model == "claude-instant-1"): + # call anthropic /completion, only use this route for claude-2, claude-instant-1 + api_base = ( + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or "https://api.anthropic.com/v1/complete" + ) + response = anthropic_text_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + ) + else: + # call /messages + # default route for all anthropic models + api_base = ( + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or "https://api.anthropic.com/v1/messages" + ) + response = anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + ) + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + elif custom_llm_provider == "nlp_cloud": + nlp_cloud_key = ( + api_key + or litellm.nlp_cloud_key + or get_secret("NLP_CLOUD_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("NLP_CLOUD_API_BASE") + or "https://api.nlpcloud.io/v1/gpu/" + ) + + response = nlp_cloud.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=nlp_cloud_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="nlp_cloud", + logging_obj=logging, + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + + response = response + elif custom_llm_provider == "aleph_alpha": + aleph_alpha_key = ( + api_key + or litellm.aleph_alpha_key + or get_secret("ALEPH_ALPHA_API_KEY") + or get_secret("ALEPHALPHA_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("ALEPH_ALPHA_API_BASE") + or "https://api.aleph-alpha.com/complete" + ) + + model_response = aleph_alpha.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + default_max_tokens_to_sample=litellm.max_tokens, + api_key=aleph_alpha_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="aleph_alpha", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "cohere": + cohere_key = ( + api_key + or litellm.cohere_key + or get_secret("COHERE_API_KEY") + or get_secret("CO_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("COHERE_API_BASE") + or "https://api.cohere.ai/v1/generate" + ) + + model_response = cohere.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=cohere_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="cohere", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "cohere_chat": + cohere_key = ( + api_key + or litellm.cohere_key + or get_secret("COHERE_API_KEY") + or get_secret("CO_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("COHERE_API_BASE") + or "https://api.cohere.ai/v1/chat" + ) + + model_response = cohere_chat.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=cohere_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="cohere_chat", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "maritalk": + maritalk_key = ( + api_key + or litellm.maritalk_key + or get_secret("MARITALK_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("MARITALK_API_BASE") + or "https://chat.maritaca.ai/api/chat/inference" + ) + + model_response = maritalk.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=maritalk_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="maritalk", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "huggingface": + custom_llm_provider = "huggingface" + huggingface_key = ( + api_key + or litellm.huggingface_key + or os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_API_KEY") + or litellm.api_key + ) + hf_headers = headers or litellm.headers + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = huggingface.completion( + model=model, + messages=messages, + api_base=api_base, # type: ignore + headers=hf_headers, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=huggingface_key, + acompletion=acompletion, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, # type: ignore + ) + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion is False + ): + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="huggingface", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "oobabooga": + custom_llm_provider = "oobabooga" + model_response = oobabooga.completion( + model=model, + messages=messages, + model_response=model_response, + api_base=api_base, # type: ignore + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=None, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + ) + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="oobabooga", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "openrouter": + api_base = api_base or litellm.api_base or "https://openrouter.ai/api/v1" + + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret("OPENROUTER_API_KEY") + or get_secret("OR_API_KEY") + ) + + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" + + headers = ( + headers + or litellm.headers + or { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + ) + + ## Load Config + config = openrouter.OpenrouterConfig.get_config() + for k, v in config.items(): + if k == "extra_body": + # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + data = {"model": model, "messages": messages, **optional_params} + + ## COMPLETION CALL + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + ) + ## LOGGING + logging.post_call( + input=messages, api_key=openai.api_key, original_response=response + ) + elif ( + custom_llm_provider == "together_ai" + or ("togethercomputer" in model) + or (model in litellm.together_ai_models) + ): + """ + Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility + """ + custom_llm_provider = "together_ai" + together_ai_key = ( + api_key + or litellm.togetherai_api_key + or get_secret("TOGETHER_AI_TOKEN") + or get_secret("TOGETHERAI_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("TOGETHERAI_API_BASE") + or "https://api.together.xyz/inference" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = together_ai.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=together_ai_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + ) + if ( + "stream_tokens" in optional_params + and optional_params["stream_tokens"] == True + ): + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="together_ai", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "palm": + palm_api_key = api_key or get_secret("PALM_API_KEY") or litellm.api_key + + # palm does not support streaming as yet :( + model_response = palm.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=palm_api_key, + logging_obj=logging, + ) + # fake palm streaming + if "stream" in optional_params and optional_params["stream"] == True: + # fake streaming for palm + resp_string = model_response["choices"][0]["message"]["content"] + response = CustomStreamWrapper( + resp_string, model, custom_llm_provider="palm", logging_obj=logging + ) + return response + response = model_response + elif custom_llm_provider == "gemini": + gemini_api_key = ( + api_key + or get_secret("GEMINI_API_KEY") + or get_secret("PALM_API_KEY") # older palm api key should also work + or litellm.api_key + ) + + # palm does not support streaming as yet :( + model_response = gemini.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=gemini_api_key, + logging_obj=logging, + acompletion=acompletion, + custom_prompt_dict=custom_prompt_dict, + ) + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion == False + ): + response = CustomStreamWrapper( + iter(model_response), + model, + custom_llm_provider="gemini", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "vertex_ai": + vertex_ai_project = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + new_params = deepcopy(optional_params) + if "claude-3" in model: + model_response = vertex_ai_anthropic.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) + else: + model_response = vertex_ai.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion == False + ): + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vertex_ai", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "predibase": + tenant_id = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) + + api_base = ( + optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.predibase_key + or get_secret("PREDIBASE_API_KEY") + ) + +> model_response = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + ) + +../main.py:1813: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +self = +model = 'llama-3-8b-instruct' +messages = [{'content': 'What is the meaning of life?', 'role': 'user'}] +api_base = None, custom_prompt_dict = {} +model_response = ModelResponse(id='chatcmpl-755fcb98-22ba-46a2-9d6d-1a85b4363e98', choices=[Choices(finish_reason='stop', index=0, mess... role='assistant'))], created=1715301477, model=None, object='chat.completion', system_fingerprint=None, usage=Usage()) +print_verbose = +encoding = , api_key = 'pb_Qg9YbQo7UqqHdu0ozxN_aw' +logging_obj = +optional_params = {'details': True, 'max_new_tokens': 256, 'return_full_text': False} +tenant_id = 'c4768f95', acompletion = False +litellm_params = {'acompletion': False, 'api_base': 'https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate_stream', 'api_key': 'pb_Qg9YbQo7UqqHdu0ozxN_aw', 'completion_call_id': None, ...} +logger_fn = None +headers = {'Authorization': 'Bearer pb_Qg9YbQo7UqqHdu0ozxN_aw', 'content-type': 'application/json'} + + def completion( + self, + model: str, + messages: list, + api_base: str, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + api_key: str, + logging_obj, + optional_params: dict, + tenant_id: str, + acompletion=None, + litellm_params=None, + logger_fn=None, + headers: dict = {}, + ) -> Union[ModelResponse, CustomStreamWrapper]: + headers = self.validate_environment(api_key, headers) + completion_url = "" + input_text = "" + base_url = "https://serving.app.predibase.com" + if "https" in model: + completion_url = model + elif api_base: + base_url = api_base + elif "PREDIBASE_API_BASE" in os.environ: + base_url = os.getenv("PREDIBASE_API_BASE", "") + + completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}" + + if optional_params.get("stream", False) == True: + completion_url += "/generate_stream" + else: + completion_url += "/generate" + + if model in custom_prompt_dict: + # check if the model has a registered custom prompt + model_prompt_details = custom_prompt_dict[model] + prompt = custom_prompt( + role_dict=model_prompt_details["roles"], + initial_prompt_value=model_prompt_details["initial_prompt_value"], + final_prompt_value=model_prompt_details["final_prompt_value"], + messages=messages, + ) + else: + prompt = prompt_factory(model=model, messages=messages) + + ## Load Config + config = litellm.PredibaseConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + stream = optional_params.pop("stream", False) + + data = { + "inputs": prompt, + "parameters": optional_params, + } + input_text = prompt + ## LOGGING + logging_obj.pre_call( + input=input_text, + api_key=api_key, + additional_args={ + "complete_input_dict": data, + "headers": headers, + "api_base": completion_url, + "acompletion": acompletion, + }, + ) + ## COMPLETION CALL + if acompletion is True: + ### ASYNC STREAMING + if stream == True: + return self.async_streaming( + model=model, + messages=messages, + data=data, + api_base=completion_url, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) # type: ignore + else: + ### ASYNC COMPLETION + return self.async_completion( + model=model, + messages=messages, + data=data, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + encoding=encoding, + api_key=api_key, + logging_obj=logging_obj, + optional_params=optional_params, + stream=False, + litellm_params=litellm_params, + logger_fn=logger_fn, + headers=headers, + ) # type: ignore + + ### SYNC STREAMING + if stream == True: + response = requests.post( + completion_url, + headers=headers, + data=json.dumps(data), +> stream=optional_params["stream"], + ) +E KeyError: 'stream' + +../llms/predibase.py:412: KeyError + +During handling of the above exception, another exception occurred: + +sync_mode = True + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_completion_predibase_streaming(sync_mode): + try: + litellm.set_verbose = True + + if sync_mode: +> response = completion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + stream=True, + ) + +test_streaming.py:317: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +args = () +kwargs = {'api_base': 'https://serving.app.predibase.com', 'api_key': 'pb_Qg9YbQo7UqqHdu0ozxN_aw', 'litellm_call_id': 'cf0ea464-1b45-4473-8e55-6bf6809df7a7', 'litellm_logging_obj': , ...} +result = None, start_time = datetime.datetime(2024, 5, 9, 17, 37, 57, 884661) +logging_obj = +call_type = 'completion', model = 'predibase/llama-3-8b-instruct' +k = 'litellm_logging_obj' + + @wraps(original_function) + def wrapper(*args, **kwargs): + # DO NOT MOVE THIS. It always needs to run first + # Check if this is an async function. If so only execute the async function + if ( + kwargs.get("acompletion", False) == True + or kwargs.get("aembedding", False) == True + or kwargs.get("aimg_generation", False) == True + or kwargs.get("amoderation", False) == True + or kwargs.get("atext_completion", False) == True + or kwargs.get("atranscription", False) == True + ): + # [OPTIONAL] CHECK MAX RETRIES / REQUEST + if litellm.num_retries_per_request is not None: + # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] + previous_models = kwargs.get("metadata", {}).get( + "previous_models", None + ) + if previous_models is not None: + if litellm.num_retries_per_request <= len(previous_models): + raise Exception(f"Max retries per request hit!") + + # MODEL CALL + result = original_function(*args, **kwargs) + if "stream" in kwargs and kwargs["stream"] == True: + if ( + "complete_response" in kwargs + and kwargs["complete_response"] == True + ): + chunks = [] + for idx, chunk in enumerate(result): + chunks.append(chunk) + return litellm.stream_chunk_builder( + chunks, messages=kwargs.get("messages", None) + ) + else: + return result + return result + + # Prints Exactly what was passed to litellm function - don't execute any logic here - it should just print + print_args_passed_to_litellm(original_function, args, kwargs) + start_time = datetime.datetime.now() + result = None + logging_obj = kwargs.get("litellm_logging_obj", None) + + # only set litellm_call_id if its not in kwargs + call_type = original_function.__name__ + if "litellm_call_id" not in kwargs: + kwargs["litellm_call_id"] = str(uuid.uuid4()) + try: + model = args[0] if len(args) > 0 else kwargs["model"] + except: + model = None + if ( + call_type != CallTypes.image_generation.value + and call_type != CallTypes.text_completion.value + ): + raise ValueError("model param not passed in.") + + try: + if logging_obj is None: + logging_obj, kwargs = function_setup( + original_function.__name__, rules_obj, start_time, *args, **kwargs + ) + kwargs["litellm_logging_obj"] = logging_obj + + # CHECK FOR 'os.environ/' in kwargs + for k, v in kwargs.items(): + if v is not None and isinstance(v, str) and v.startswith("os.environ/"): + kwargs[k] = litellm.get_secret(v) + # [OPTIONAL] CHECK BUDGET + if litellm.max_budget: + if litellm._current_cost > litellm.max_budget: + raise BudgetExceededError( + current_cost=litellm._current_cost, + max_budget=litellm.max_budget, + ) + + # [OPTIONAL] CHECK MAX RETRIES / REQUEST + if litellm.num_retries_per_request is not None: + # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] + previous_models = kwargs.get("metadata", {}).get( + "previous_models", None + ) + if previous_models is not None: + if litellm.num_retries_per_request <= len(previous_models): + raise Exception(f"Max retries per request hit!") + + # [OPTIONAL] CHECK CACHE + print_verbose( + f"SYNC kwargs[caching]: {kwargs.get('caching', False)}; litellm.cache: {litellm.cache}; kwargs.get('cache')['no-cache']: {kwargs.get('cache', {}).get('no-cache', False)}" + ) + # if caching is false or cache["no-cache"]==True, don't run this + if ( + ( + ( + ( + kwargs.get("caching", None) is None + and litellm.cache is not None + ) + or kwargs.get("caching", False) == True + ) + and kwargs.get("cache", {}).get("no-cache", False) != True + ) + and kwargs.get("aembedding", False) != True + and kwargs.get("atext_completion", False) != True + and kwargs.get("acompletion", False) != True + and kwargs.get("aimg_generation", False) != True + and kwargs.get("atranscription", False) != True + ): # allow users to control returning cached responses from the completion function + # checking cache + print_verbose(f"INSIDE CHECKING CACHE") + if ( + litellm.cache is not None + and str(original_function.__name__) + in litellm.cache.supported_call_types + ): + print_verbose(f"Checking Cache") + preset_cache_key = litellm.cache.get_cache_key(*args, **kwargs) + kwargs["preset_cache_key"] = ( + preset_cache_key # for streaming calls, we need to pass the preset_cache_key + ) + cached_result = litellm.cache.get_cache(*args, **kwargs) + if cached_result != None: + if "detail" in cached_result: + # implies an error occurred + pass + else: + call_type = original_function.__name__ + print_verbose( + f"Cache Response Object routing: call_type - {call_type}; cached_result instace: {type(cached_result)}" + ) + if call_type == CallTypes.completion.value and isinstance( + cached_result, dict + ): + cached_result = convert_to_model_response_object( + response_object=cached_result, + model_response_object=ModelResponse(), + stream=kwargs.get("stream", False), + ) + if kwargs.get("stream", False) == True: + cached_result = CustomStreamWrapper( + completion_stream=cached_result, + model=model, + custom_llm_provider="cached_response", + logging_obj=logging_obj, + ) + elif call_type == CallTypes.embedding.value and isinstance( + cached_result, dict + ): + cached_result = convert_to_model_response_object( + response_object=cached_result, + response_type="embedding", + ) + + # LOG SUCCESS + cache_hit = True + end_time = datetime.datetime.now() + ( + model, + custom_llm_provider, + dynamic_api_key, + api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=kwargs.get( + "custom_llm_provider", None + ), + api_base=kwargs.get("api_base", None), + api_key=kwargs.get("api_key", None), + ) + print_verbose( + f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" + ) + logging_obj.update_environment_variables( + model=model, + user=kwargs.get("user", None), + optional_params={}, + litellm_params={ + "logger_fn": kwargs.get("logger_fn", None), + "acompletion": False, + "metadata": kwargs.get("metadata", {}), + "model_info": kwargs.get("model_info", {}), + "proxy_server_request": kwargs.get( + "proxy_server_request", None + ), + "preset_cache_key": kwargs.get( + "preset_cache_key", None + ), + "stream_response": kwargs.get( + "stream_response", {} + ), + }, + input=kwargs.get("messages", ""), + api_key=kwargs.get("api_key", None), + original_response=str(cached_result), + additional_args=None, + stream=kwargs.get("stream", False), + ) + threading.Thread( + target=logging_obj.success_handler, + args=(cached_result, start_time, end_time, cache_hit), + ).start() + return cached_result + + # CHECK MAX TOKENS + if ( + kwargs.get("max_tokens", None) is not None + and model is not None + and litellm.modify_params + == True # user is okay with params being modified + and ( + call_type == CallTypes.acompletion.value + or call_type == CallTypes.completion.value + ) + ): + try: + base_model = model + if kwargs.get("hf_model_name", None) is not None: + base_model = f"huggingface/{kwargs.get('hf_model_name')}" + max_output_tokens = ( + get_max_tokens(model=base_model) or 4096 + ) # assume min context window is 4k tokens + user_max_tokens = kwargs.get("max_tokens") + ## Scenario 1: User limit + prompt > model limit + messages = None + if len(args) > 1: + messages = args[1] + elif kwargs.get("messages", None): + messages = kwargs["messages"] + input_tokens = token_counter(model=base_model, messages=messages) + input_tokens += max( + 0.1 * input_tokens, 10 + ) # give at least a 10 token buffer. token counting can be imprecise. + if input_tokens > max_output_tokens: + pass # allow call to fail normally + elif user_max_tokens + input_tokens > max_output_tokens: + user_max_tokens = max_output_tokens - input_tokens + print_verbose(f"user_max_tokens: {user_max_tokens}") + kwargs["max_tokens"] = int( + round(user_max_tokens) + ) # make sure max tokens is always an int + except Exception as e: + print_verbose(f"Error while checking max token limit: {str(e)}") + # MODEL CALL + result = original_function(*args, **kwargs) + end_time = datetime.datetime.now() + if "stream" in kwargs and kwargs["stream"] == True: + if ( + "complete_response" in kwargs + and kwargs["complete_response"] == True + ): + chunks = [] + for idx, chunk in enumerate(result): + chunks.append(chunk) + return litellm.stream_chunk_builder( + chunks, messages=kwargs.get("messages", None) + ) + else: + return result + elif "acompletion" in kwargs and kwargs["acompletion"] == True: + return result + elif "aembedding" in kwargs and kwargs["aembedding"] == True: + return result + elif "aimg_generation" in kwargs and kwargs["aimg_generation"] == True: + return result + elif "atranscription" in kwargs and kwargs["atranscription"] == True: + return result + + ### POST-CALL RULES ### + post_call_processing(original_response=result, model=model or None) + + # [OPTIONAL] ADD TO CACHE + if ( + litellm.cache is not None + and str(original_function.__name__) + in litellm.cache.supported_call_types + ) and (kwargs.get("cache", {}).get("no-store", False) != True): + litellm.cache.add_cache(result, *args, **kwargs) + + # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated + verbose_logger.info(f"Wrapper: Completed Call, calling success_handler") + threading.Thread( + target=logging_obj.success_handler, args=(result, start_time, end_time) + ).start() + # RETURN RESULT + if hasattr(result, "_hidden_params"): + result._hidden_params["model_id"] = kwargs.get("model_info", {}).get( + "id", None + ) + result._hidden_params["api_base"] = get_api_base( + model=model, + optional_params=getattr(logging_obj, "optional_params", {}), + ) + result._response_ms = ( + end_time - start_time + ).total_seconds() * 1000 # return response latency in ms like openai + return result + except Exception as e: + call_type = original_function.__name__ + if call_type == CallTypes.completion.value: + num_retries = ( + kwargs.get("num_retries", None) or litellm.num_retries or None + ) + litellm.num_retries = ( + None # set retries to None to prevent infinite loops + ) + context_window_fallback_dict = kwargs.get( + "context_window_fallback_dict", {} + ) + + _is_litellm_router_call = "model_group" in kwargs.get( + "metadata", {} + ) # check if call from litellm.router/proxy + if ( + num_retries and not _is_litellm_router_call + ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying + if ( + isinstance(e, openai.APIError) + or isinstance(e, openai.Timeout) + or isinstance(e, openai.APIConnectionError) + ): + kwargs["num_retries"] = num_retries + return litellm.completion_with_retries(*args, **kwargs) + elif ( + isinstance(e, litellm.exceptions.ContextWindowExceededError) + and context_window_fallback_dict + and model in context_window_fallback_dict + ): + if len(args) > 0: + args[0] = context_window_fallback_dict[model] + else: + kwargs["model"] = context_window_fallback_dict[model] + return original_function(*args, **kwargs) + traceback_exception = traceback.format_exc() + end_time = datetime.datetime.now() + # LOG FAILURE - handle streaming failure logging in the _next_ object, remove `handle_failure` once it's deprecated + if logging_obj: + logging_obj.failure_handler( + e, traceback_exception, start_time, end_time + ) # DO NOT MAKE THREADED - router retry fallback relies on this! + my_thread = threading.Thread( + target=handle_failure, + args=(e, traceback_exception, start_time, end_time, args, kwargs), + ) # don't interrupt execution of main thread + my_thread.start() + if hasattr(e, "message"): + if ( + liteDebuggerClient and liteDebuggerClient.dashboard_url != None + ): # make it easy to get to the debugger logs if you've initialized it + e.message += f"\n Check the log in your dashboard - {liteDebuggerClient.dashboard_url}" +> raise e + +../utils.py:3229: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +args = () +kwargs = {'api_base': 'https://serving.app.predibase.com', 'api_key': 'pb_Qg9YbQo7UqqHdu0ozxN_aw', 'litellm_call_id': 'cf0ea464-1b45-4473-8e55-6bf6809df7a7', 'litellm_logging_obj': , ...} +result = None, start_time = datetime.datetime(2024, 5, 9, 17, 37, 57, 884661) +logging_obj = +call_type = 'completion', model = 'predibase/llama-3-8b-instruct' +k = 'litellm_logging_obj' + + @wraps(original_function) + def wrapper(*args, **kwargs): + # DO NOT MOVE THIS. It always needs to run first + # Check if this is an async function. If so only execute the async function + if ( + kwargs.get("acompletion", False) == True + or kwargs.get("aembedding", False) == True + or kwargs.get("aimg_generation", False) == True + or kwargs.get("amoderation", False) == True + or kwargs.get("atext_completion", False) == True + or kwargs.get("atranscription", False) == True + ): + # [OPTIONAL] CHECK MAX RETRIES / REQUEST + if litellm.num_retries_per_request is not None: + # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] + previous_models = kwargs.get("metadata", {}).get( + "previous_models", None + ) + if previous_models is not None: + if litellm.num_retries_per_request <= len(previous_models): + raise Exception(f"Max retries per request hit!") + + # MODEL CALL + result = original_function(*args, **kwargs) + if "stream" in kwargs and kwargs["stream"] == True: + if ( + "complete_response" in kwargs + and kwargs["complete_response"] == True + ): + chunks = [] + for idx, chunk in enumerate(result): + chunks.append(chunk) + return litellm.stream_chunk_builder( + chunks, messages=kwargs.get("messages", None) + ) + else: + return result + return result + + # Prints Exactly what was passed to litellm function - don't execute any logic here - it should just print + print_args_passed_to_litellm(original_function, args, kwargs) + start_time = datetime.datetime.now() + result = None + logging_obj = kwargs.get("litellm_logging_obj", None) + + # only set litellm_call_id if its not in kwargs + call_type = original_function.__name__ + if "litellm_call_id" not in kwargs: + kwargs["litellm_call_id"] = str(uuid.uuid4()) + try: + model = args[0] if len(args) > 0 else kwargs["model"] + except: + model = None + if ( + call_type != CallTypes.image_generation.value + and call_type != CallTypes.text_completion.value + ): + raise ValueError("model param not passed in.") + + try: + if logging_obj is None: + logging_obj, kwargs = function_setup( + original_function.__name__, rules_obj, start_time, *args, **kwargs + ) + kwargs["litellm_logging_obj"] = logging_obj + + # CHECK FOR 'os.environ/' in kwargs + for k, v in kwargs.items(): + if v is not None and isinstance(v, str) and v.startswith("os.environ/"): + kwargs[k] = litellm.get_secret(v) + # [OPTIONAL] CHECK BUDGET + if litellm.max_budget: + if litellm._current_cost > litellm.max_budget: + raise BudgetExceededError( + current_cost=litellm._current_cost, + max_budget=litellm.max_budget, + ) + + # [OPTIONAL] CHECK MAX RETRIES / REQUEST + if litellm.num_retries_per_request is not None: + # check if previous_models passed in as ['litellm_params']['metadata]['previous_models'] + previous_models = kwargs.get("metadata", {}).get( + "previous_models", None + ) + if previous_models is not None: + if litellm.num_retries_per_request <= len(previous_models): + raise Exception(f"Max retries per request hit!") + + # [OPTIONAL] CHECK CACHE + print_verbose( + f"SYNC kwargs[caching]: {kwargs.get('caching', False)}; litellm.cache: {litellm.cache}; kwargs.get('cache')['no-cache']: {kwargs.get('cache', {}).get('no-cache', False)}" + ) + # if caching is false or cache["no-cache"]==True, don't run this + if ( + ( + ( + ( + kwargs.get("caching", None) is None + and litellm.cache is not None + ) + or kwargs.get("caching", False) == True + ) + and kwargs.get("cache", {}).get("no-cache", False) != True + ) + and kwargs.get("aembedding", False) != True + and kwargs.get("atext_completion", False) != True + and kwargs.get("acompletion", False) != True + and kwargs.get("aimg_generation", False) != True + and kwargs.get("atranscription", False) != True + ): # allow users to control returning cached responses from the completion function + # checking cache + print_verbose(f"INSIDE CHECKING CACHE") + if ( + litellm.cache is not None + and str(original_function.__name__) + in litellm.cache.supported_call_types + ): + print_verbose(f"Checking Cache") + preset_cache_key = litellm.cache.get_cache_key(*args, **kwargs) + kwargs["preset_cache_key"] = ( + preset_cache_key # for streaming calls, we need to pass the preset_cache_key + ) + cached_result = litellm.cache.get_cache(*args, **kwargs) + if cached_result != None: + if "detail" in cached_result: + # implies an error occurred + pass + else: + call_type = original_function.__name__ + print_verbose( + f"Cache Response Object routing: call_type - {call_type}; cached_result instace: {type(cached_result)}" + ) + if call_type == CallTypes.completion.value and isinstance( + cached_result, dict + ): + cached_result = convert_to_model_response_object( + response_object=cached_result, + model_response_object=ModelResponse(), + stream=kwargs.get("stream", False), + ) + if kwargs.get("stream", False) == True: + cached_result = CustomStreamWrapper( + completion_stream=cached_result, + model=model, + custom_llm_provider="cached_response", + logging_obj=logging_obj, + ) + elif call_type == CallTypes.embedding.value and isinstance( + cached_result, dict + ): + cached_result = convert_to_model_response_object( + response_object=cached_result, + response_type="embedding", + ) + + # LOG SUCCESS + cache_hit = True + end_time = datetime.datetime.now() + ( + model, + custom_llm_provider, + dynamic_api_key, + api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=kwargs.get( + "custom_llm_provider", None + ), + api_base=kwargs.get("api_base", None), + api_key=kwargs.get("api_key", None), + ) + print_verbose( + f"Async Wrapper: Completed Call, calling async_success_handler: {logging_obj.async_success_handler}" + ) + logging_obj.update_environment_variables( + model=model, + user=kwargs.get("user", None), + optional_params={}, + litellm_params={ + "logger_fn": kwargs.get("logger_fn", None), + "acompletion": False, + "metadata": kwargs.get("metadata", {}), + "model_info": kwargs.get("model_info", {}), + "proxy_server_request": kwargs.get( + "proxy_server_request", None + ), + "preset_cache_key": kwargs.get( + "preset_cache_key", None + ), + "stream_response": kwargs.get( + "stream_response", {} + ), + }, + input=kwargs.get("messages", ""), + api_key=kwargs.get("api_key", None), + original_response=str(cached_result), + additional_args=None, + stream=kwargs.get("stream", False), + ) + threading.Thread( + target=logging_obj.success_handler, + args=(cached_result, start_time, end_time, cache_hit), + ).start() + return cached_result + + # CHECK MAX TOKENS + if ( + kwargs.get("max_tokens", None) is not None + and model is not None + and litellm.modify_params + == True # user is okay with params being modified + and ( + call_type == CallTypes.acompletion.value + or call_type == CallTypes.completion.value + ) + ): + try: + base_model = model + if kwargs.get("hf_model_name", None) is not None: + base_model = f"huggingface/{kwargs.get('hf_model_name')}" + max_output_tokens = ( + get_max_tokens(model=base_model) or 4096 + ) # assume min context window is 4k tokens + user_max_tokens = kwargs.get("max_tokens") + ## Scenario 1: User limit + prompt > model limit + messages = None + if len(args) > 1: + messages = args[1] + elif kwargs.get("messages", None): + messages = kwargs["messages"] + input_tokens = token_counter(model=base_model, messages=messages) + input_tokens += max( + 0.1 * input_tokens, 10 + ) # give at least a 10 token buffer. token counting can be imprecise. + if input_tokens > max_output_tokens: + pass # allow call to fail normally + elif user_max_tokens + input_tokens > max_output_tokens: + user_max_tokens = max_output_tokens - input_tokens + print_verbose(f"user_max_tokens: {user_max_tokens}") + kwargs["max_tokens"] = int( + round(user_max_tokens) + ) # make sure max tokens is always an int + except Exception as e: + print_verbose(f"Error while checking max token limit: {str(e)}") + # MODEL CALL +> result = original_function(*args, **kwargs) + +../utils.py:3123: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +model = 'llama-3-8b-instruct' +messages = [{'content': 'What is the meaning of life?', 'role': 'user'}] +timeout = 600.0, temperature = None, top_p = None, n = None, stream = True +stream_options = None, stop = None, max_tokens = None, presence_penalty = None +frequency_penalty = None, logit_bias = None, user = None, response_format = None +seed = None, tools = None, tool_choice = None, logprobs = None +top_logprobs = None, deployment_id = None, extra_headers = None +functions = None, function_call = None, base_url = None, api_version = None +api_key = 'pb_Qg9YbQo7UqqHdu0ozxN_aw', model_list = None +kwargs = {'api_base': 'https://serving.app.predibase.com', 'litellm_call_id': 'cf0ea464-1b45-4473-8e55-6bf6809df7a7', 'litellm_logging_obj': , 'tenant_id': 'c4768f95'} +args = {'acompletion': False, 'api_base': None, 'api_key': 'pb_Qg9YbQo7UqqHdu0ozxN_aw', 'api_version': None, ...} +api_base = None, mock_response = None, force_timeout = 600, logger_fn = None +verbose = False, custom_llm_provider = 'predibase' + + @client + def completion( + model: str, + # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create + messages: List = [], + timeout: Optional[Union[float, str, httpx.Timeout]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + n: Optional[int] = None, + stream: Optional[bool] = None, + stream_options: Optional[dict] = None, + stop=None, + max_tokens: Optional[int] = None, + presence_penalty: Optional[float] = None, + frequency_penalty: Optional[float] = None, + logit_bias: Optional[dict] = None, + user: Optional[str] = None, + # openai v1.0+ new params + response_format: Optional[dict] = None, + seed: Optional[int] = None, + tools: Optional[List] = None, + tool_choice: Optional[str] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + deployment_id=None, + extra_headers: Optional[dict] = None, + # soon to be deprecated params by OpenAI + functions: Optional[List] = None, + function_call: Optional[str] = None, + # set api_base, api_version, api_key + base_url: Optional[str] = None, + api_version: Optional[str] = None, + api_key: Optional[str] = None, + model_list: Optional[list] = None, # pass in a list of api_base,keys, etc. + # Optional liteLLM function params + **kwargs, + ) -> Union[ModelResponse, CustomStreamWrapper]: + """ + Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly) + Parameters: + model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/ + messages (List): A list of message objects representing the conversation context (default is an empty list). + + OPTIONAL PARAMS + functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list). + function_call (str, optional): The name of the function to call within the conversation (default is an empty string). + temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0). + top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0). + n (int, optional): The number of completions to generate (default is 1). + stream (bool, optional): If True, return a streaming response (default is False). + stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true. + stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens. + max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity). + presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far. + frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far. + logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion. + user (str, optional): A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse. + logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message + top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used. + metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc. + api_base (str, optional): Base URL for the API (default is None). + api_version (str, optional): API version (default is None). + api_key (str, optional): API key (default is None). + model_list (list, optional): List of api base, version, keys + extra_headers (dict, optional): Additional headers to include in the request. + + LITELLM Specific Params + mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None). + custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock" + max_retries (int, optional): The number of retries to attempt (default is 0). + Returns: + ModelResponse: A response object containing the generated completion and associated metadata. + + Note: + - This function is used to perform completions() using the specified language model. + - It supports various optional parameters for customizing the completion behavior. + - If 'mock_response' is provided, a mock completion response is returned for testing or debugging. + """ + ######### unpacking kwargs ##################### + args = locals() + api_base = kwargs.get("api_base", None) + mock_response = kwargs.get("mock_response", None) + force_timeout = kwargs.get("force_timeout", 600) ## deprecated + logger_fn = kwargs.get("logger_fn", None) + verbose = kwargs.get("verbose", False) + custom_llm_provider = kwargs.get("custom_llm_provider", None) + litellm_logging_obj = kwargs.get("litellm_logging_obj", None) + id = kwargs.get("id", None) + metadata = kwargs.get("metadata", None) + model_info = kwargs.get("model_info", None) + proxy_server_request = kwargs.get("proxy_server_request", None) + fallbacks = kwargs.get("fallbacks", None) + headers = kwargs.get("headers", None) + num_retries = kwargs.get("num_retries", None) ## deprecated + max_retries = kwargs.get("max_retries", None) + context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None) + organization = kwargs.get("organization", None) + ### CUSTOM MODEL COST ### + input_cost_per_token = kwargs.get("input_cost_per_token", None) + output_cost_per_token = kwargs.get("output_cost_per_token", None) + input_cost_per_second = kwargs.get("input_cost_per_second", None) + output_cost_per_second = kwargs.get("output_cost_per_second", None) + ### CUSTOM PROMPT TEMPLATE ### + initial_prompt_value = kwargs.get("initial_prompt_value", None) + roles = kwargs.get("roles", None) + final_prompt_value = kwargs.get("final_prompt_value", None) + bos_token = kwargs.get("bos_token", None) + eos_token = kwargs.get("eos_token", None) + preset_cache_key = kwargs.get("preset_cache_key", None) + hf_model_name = kwargs.get("hf_model_name", None) + supports_system_message = kwargs.get("supports_system_message", None) + ### TEXT COMPLETION CALLS ### + text_completion = kwargs.get("text_completion", False) + atext_completion = kwargs.get("atext_completion", False) + ### ASYNC CALLS ### + acompletion = kwargs.get("acompletion", False) + client = kwargs.get("client", None) + ### Admin Controls ### + no_log = kwargs.get("no-log", False) + ######## end of unpacking kwargs ########### + openai_params = [ + "functions", + "function_call", + "temperature", + "temperature", + "top_p", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "request_timeout", + "api_base", + "api_version", + "api_key", + "deployment_id", + "organization", + "base_url", + "default_headers", + "timeout", + "response_format", + "seed", + "tools", + "tool_choice", + "max_retries", + "logprobs", + "top_logprobs", + "extra_headers", + ] + litellm_params = [ + "metadata", + "acompletion", + "atext_completion", + "text_completion", + "caching", + "mock_response", + "api_key", + "api_version", + "api_base", + "force_timeout", + "logger_fn", + "verbose", + "custom_llm_provider", + "litellm_logging_obj", + "litellm_call_id", + "use_client", + "id", + "fallbacks", + "azure", + "headers", + "model_list", + "num_retries", + "context_window_fallback_dict", + "retry_policy", + "roles", + "final_prompt_value", + "bos_token", + "eos_token", + "request_timeout", + "complete_response", + "self", + "client", + "rpm", + "tpm", + "max_parallel_requests", + "input_cost_per_token", + "output_cost_per_token", + "input_cost_per_second", + "output_cost_per_second", + "hf_model_name", + "model_info", + "proxy_server_request", + "preset_cache_key", + "caching_groups", + "ttl", + "cache", + "no-log", + "base_model", + "stream_timeout", + "supports_system_message", + "region_name", + "allowed_model_region", + ] + default_params = openai_params + litellm_params + non_default_params = { + k: v for k, v in kwargs.items() if k not in default_params + } # model-specific params - pass them straight to the model/provider + + ### TIMEOUT LOGIC ### + timeout = timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + + try: + if base_url is not None: + api_base = base_url + if max_retries is not None: # openai allows openai.OpenAI(max_retries=3) + num_retries = max_retries + logging = litellm_logging_obj + fallbacks = fallbacks or litellm.model_fallbacks + if fallbacks is not None: + return completion_with_fallbacks(**args) + if model_list is not None: + deployments = [ + m["litellm_params"] for m in model_list if m["model_name"] == model + ] + return batch_completion_models(deployments=deployments, **args) + if litellm.model_alias_map and model in litellm.model_alias_map: + model = litellm.model_alias_map[ + model + ] # update the model to the actual value if an alias has been passed in + model_response = ModelResponse() + setattr(model_response, "usage", litellm.Usage()) + if ( + kwargs.get("azure", False) == True + ): # don't remove flag check, to remain backwards compatible for repos like Codium + custom_llm_provider = "azure" + if deployment_id != None: # azure llms + model = deployment_id + custom_llm_provider = "azure" + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + if model_response is not None and hasattr(model_response, "_hidden_params"): + model_response._hidden_params["custom_llm_provider"] = custom_llm_provider + model_response._hidden_params["region_name"] = kwargs.get( + "aws_region_name", None + ) # support region-based pricing for bedrock + + ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### + if input_cost_per_token is not None and output_cost_per_token is not None: + print_verbose(f"Registering model={model} in model cost map") + litellm.register_model( + { + f"{custom_llm_provider}/{model}": { + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + "litellm_provider": custom_llm_provider, + }, + model: { + "input_cost_per_token": input_cost_per_token, + "output_cost_per_token": output_cost_per_token, + "litellm_provider": custom_llm_provider, + }, + } + ) + elif ( + input_cost_per_second is not None + ): # time based pricing just needs cost in place + output_cost_per_second = output_cost_per_second + litellm.register_model( + { + f"{custom_llm_provider}/{model}": { + "input_cost_per_second": input_cost_per_second, + "output_cost_per_second": output_cost_per_second, + "litellm_provider": custom_llm_provider, + }, + model: { + "input_cost_per_second": input_cost_per_second, + "output_cost_per_second": output_cost_per_second, + "litellm_provider": custom_llm_provider, + }, + } + ) + ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### + custom_prompt_dict = {} # type: ignore + if ( + initial_prompt_value + or roles + or final_prompt_value + or bos_token + or eos_token + ): + custom_prompt_dict = {model: {}} + if initial_prompt_value: + custom_prompt_dict[model]["initial_prompt_value"] = initial_prompt_value + if roles: + custom_prompt_dict[model]["roles"] = roles + if final_prompt_value: + custom_prompt_dict[model]["final_prompt_value"] = final_prompt_value + if bos_token: + custom_prompt_dict[model]["bos_token"] = bos_token + if eos_token: + custom_prompt_dict[model]["eos_token"] = eos_token + + if ( + supports_system_message is not None + and isinstance(supports_system_message, bool) + and supports_system_message == False + ): + messages = map_system_message_pt(messages=messages) + model_api_key = get_api_key( + llm_provider=custom_llm_provider, dynamic_api_key=api_key + ) # get the api key from the environment if required for the model + + if dynamic_api_key is not None: + api_key = dynamic_api_key + # check if user passed in any of the OpenAI optional params + optional_params = get_optional_params( + functions=functions, + function_call=function_call, + temperature=temperature, + top_p=top_p, + n=n, + stream=stream, + stream_options=stream_options, + stop=stop, + max_tokens=max_tokens, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + logit_bias=logit_bias, + user=user, + # params to identify the model + model=model, + custom_llm_provider=custom_llm_provider, + response_format=response_format, + seed=seed, + tools=tools, + tool_choice=tool_choice, + max_retries=max_retries, + logprobs=logprobs, + top_logprobs=top_logprobs, + extra_headers=extra_headers, + **non_default_params, + ) + + if litellm.add_function_to_prompt and optional_params.get( + "functions_unsupported_model", None + ): # if user opts to add it to prompt, when API doesn't support function calling + functions_unsupported_model = optional_params.pop( + "functions_unsupported_model" + ) + messages = function_call_prompt( + messages=messages, functions=functions_unsupported_model + ) + + # For logging - save the values of the litellm-specific params passed in + litellm_params = get_litellm_params( + acompletion=acompletion, + api_key=api_key, + force_timeout=force_timeout, + logger_fn=logger_fn, + verbose=verbose, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + litellm_call_id=kwargs.get("litellm_call_id", None), + model_alias_map=litellm.model_alias_map, + completion_call_id=id, + metadata=metadata, + model_info=model_info, + proxy_server_request=proxy_server_request, + preset_cache_key=preset_cache_key, + no_log=no_log, + ) + logging.update_environment_variables( + model=model, + user=user, + optional_params=optional_params, + litellm_params=litellm_params, + ) + if mock_response: + return mock_completion( + model, + messages, + stream=stream, + mock_response=mock_response, + logging=logging, + acompletion=acompletion, + ) + if custom_llm_provider == "azure": + # azure configs + api_type = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret("AZURE_AD_TOKEN") + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + elif custom_llm_provider == "azure_text": + # azure configs + api_type = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret("AZURE_AD_TOKEN") + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_text_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + elif ( + model in litellm.open_ai_chat_completion_models + or custom_llm_provider == "custom_openai" + or custom_llm_provider == "deepinfra" + or custom_llm_provider == "perplexity" + or custom_llm_provider == "groq" + or custom_llm_provider == "deepseek" + or custom_llm_provider == "anyscale" + or custom_llm_provider == "mistral" + or custom_llm_provider == "openai" + or custom_llm_provider == "together_ai" + or custom_llm_provider in litellm.openai_compatible_providers + or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo + ): # allow user to make an openai call with a custom base + # note: if a user sets a custom base - we should ensure this works + # allow for the setting of dynamic and stateful api-bases + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + openai.organization = ( + organization + or litellm.organization + or get_secret("OPENAI_ORGANIZATION") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.OpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + try: + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + organization=organization, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + elif ( + custom_llm_provider == "text-completion-openai" + or "ft:babbage-002" in model + or "ft:davinci-002" in model # support for finetuned completion models + ): + openai.api_type = "openai" + + api_base = ( + api_base + or litellm.api_base + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + + openai.api_version = None + # set API KEY + + api_key = ( + api_key + or litellm.api_key + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.OpenAITextCompletionConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + if litellm.organization: + openai.organization = litellm.organization + + if ( + len(messages) > 0 + and "content" in messages[0] + and type(messages[0]["content"]) == list + ): + # text-davinci-003 can accept a string or array, if it's an array, assume the array is set in messages[0]['content'] + # https://platform.openai.com/docs/api-reference/completions/create + prompt = messages[0]["content"] + else: + prompt = " ".join([message["content"] for message in messages]) # type: ignore + + ## COMPLETION CALL + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + client=client, # pass AsyncOpenAI, OpenAI client + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) + + if ( + optional_params.get("stream", False) == False + and acompletion == False + and text_completion == False + ): + # convert to chat completion response + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + response = _response + elif ( + "replicate" in model + or custom_llm_provider == "replicate" + or model in litellm.replicate_models + ): + # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") + replicate_key = None + replicate_key = ( + api_key + or litellm.replicate_key + or litellm.api_key + or get_secret("REPLICATE_API_KEY") + or get_secret("REPLICATE_API_TOKEN") + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("REPLICATE_API_BASE") + or "https://api.replicate.com/v1" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = replicate.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=replicate_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + ) + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + model_response = CustomStreamWrapper(model_response, model, logging_obj=logging, custom_llm_provider="replicate") # type: ignore + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=replicate_key, + original_response=model_response, + ) + + response = model_response + + elif custom_llm_provider == "anthropic": + api_key = ( + api_key + or litellm.anthropic_key + or litellm.api_key + or os.environ.get("ANTHROPIC_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + if (model == "claude-2") or (model == "claude-instant-1"): + # call anthropic /completion, only use this route for claude-2, claude-instant-1 + api_base = ( + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or "https://api.anthropic.com/v1/complete" + ) + response = anthropic_text_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + ) + else: + # call /messages + # default route for all anthropic models + api_base = ( + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or "https://api.anthropic.com/v1/messages" + ) + response = anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + ) + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + elif custom_llm_provider == "nlp_cloud": + nlp_cloud_key = ( + api_key + or litellm.nlp_cloud_key + or get_secret("NLP_CLOUD_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("NLP_CLOUD_API_BASE") + or "https://api.nlpcloud.io/v1/gpu/" + ) + + response = nlp_cloud.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=nlp_cloud_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="nlp_cloud", + logging_obj=logging, + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + + response = response + elif custom_llm_provider == "aleph_alpha": + aleph_alpha_key = ( + api_key + or litellm.aleph_alpha_key + or get_secret("ALEPH_ALPHA_API_KEY") + or get_secret("ALEPHALPHA_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("ALEPH_ALPHA_API_BASE") + or "https://api.aleph-alpha.com/complete" + ) + + model_response = aleph_alpha.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + default_max_tokens_to_sample=litellm.max_tokens, + api_key=aleph_alpha_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="aleph_alpha", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "cohere": + cohere_key = ( + api_key + or litellm.cohere_key + or get_secret("COHERE_API_KEY") + or get_secret("CO_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("COHERE_API_BASE") + or "https://api.cohere.ai/v1/generate" + ) + + model_response = cohere.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=cohere_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="cohere", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "cohere_chat": + cohere_key = ( + api_key + or litellm.cohere_key + or get_secret("COHERE_API_KEY") + or get_secret("CO_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("COHERE_API_BASE") + or "https://api.cohere.ai/v1/chat" + ) + + model_response = cohere_chat.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=cohere_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="cohere_chat", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "maritalk": + maritalk_key = ( + api_key + or litellm.maritalk_key + or get_secret("MARITALK_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("MARITALK_API_BASE") + or "https://chat.maritaca.ai/api/chat/inference" + ) + + model_response = maritalk.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=maritalk_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="maritalk", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "huggingface": + custom_llm_provider = "huggingface" + huggingface_key = ( + api_key + or litellm.huggingface_key + or os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_API_KEY") + or litellm.api_key + ) + hf_headers = headers or litellm.headers + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = huggingface.completion( + model=model, + messages=messages, + api_base=api_base, # type: ignore + headers=hf_headers, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=huggingface_key, + acompletion=acompletion, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, # type: ignore + ) + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion is False + ): + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="huggingface", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "oobabooga": + custom_llm_provider = "oobabooga" + model_response = oobabooga.completion( + model=model, + messages=messages, + model_response=model_response, + api_base=api_base, # type: ignore + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=None, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + ) + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="oobabooga", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "openrouter": + api_base = api_base or litellm.api_base or "https://openrouter.ai/api/v1" + + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret("OPENROUTER_API_KEY") + or get_secret("OR_API_KEY") + ) + + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" + + headers = ( + headers + or litellm.headers + or { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } + ) + + ## Load Config + config = openrouter.OpenrouterConfig.get_config() + for k, v in config.items(): + if k == "extra_body": + # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + data = {"model": model, "messages": messages, **optional_params} + + ## COMPLETION CALL + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + ) + ## LOGGING + logging.post_call( + input=messages, api_key=openai.api_key, original_response=response + ) + elif ( + custom_llm_provider == "together_ai" + or ("togethercomputer" in model) + or (model in litellm.together_ai_models) + ): + """ + Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility + """ + custom_llm_provider = "together_ai" + together_ai_key = ( + api_key + or litellm.togetherai_api_key + or get_secret("TOGETHER_AI_TOKEN") + or get_secret("TOGETHERAI_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("TOGETHERAI_API_BASE") + or "https://api.together.xyz/inference" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = together_ai.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=together_ai_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + ) + if ( + "stream_tokens" in optional_params + and optional_params["stream_tokens"] == True + ): + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="together_ai", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "palm": + palm_api_key = api_key or get_secret("PALM_API_KEY") or litellm.api_key + + # palm does not support streaming as yet :( + model_response = palm.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=palm_api_key, + logging_obj=logging, + ) + # fake palm streaming + if "stream" in optional_params and optional_params["stream"] == True: + # fake streaming for palm + resp_string = model_response["choices"][0]["message"]["content"] + response = CustomStreamWrapper( + resp_string, model, custom_llm_provider="palm", logging_obj=logging + ) + return response + response = model_response + elif custom_llm_provider == "gemini": + gemini_api_key = ( + api_key + or get_secret("GEMINI_API_KEY") + or get_secret("PALM_API_KEY") # older palm api key should also work + or litellm.api_key + ) + + # palm does not support streaming as yet :( + model_response = gemini.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=gemini_api_key, + logging_obj=logging, + acompletion=acompletion, + custom_prompt_dict=custom_prompt_dict, + ) + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion == False + ): + response = CustomStreamWrapper( + iter(model_response), + model, + custom_llm_provider="gemini", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "vertex_ai": + vertex_ai_project = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + new_params = deepcopy(optional_params) + if "claude-3" in model: + model_response = vertex_ai_anthropic.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) + else: + model_response = vertex_ai.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion == False + ): + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vertex_ai", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "predibase": + tenant_id = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) + + api_base = ( + optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.predibase_key + or get_secret("PREDIBASE_API_KEY") + ) + + model_response = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] == True + and acompletion == False + ): + return response + response = model_response + elif custom_llm_provider == "ai21": + custom_llm_provider = "ai21" + ai21_key = ( + api_key + or litellm.ai21_key + or os.environ.get("AI21_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("AI21_API_BASE") + or "https://api.ai21.com/studio/v1/" + ) + + model_response = ai21.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=ai21_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="ai21", + logging_obj=logging, + ) + return response + + ## RESPONSE OBJECT + response = model_response + elif custom_llm_provider == "sagemaker": + # boto3 reads keys from .env + model_response = sagemaker.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + custom_prompt_dict=custom_prompt_dict, + hf_model_name=hf_model_name, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + acompletion=acompletion, + ) + if ( + "stream" in optional_params and optional_params["stream"] == True + ): ## [BETA] + print_verbose(f"ENTERS SAGEMAKER CUSTOMSTREAMWRAPPER") + from .llms.sagemaker import TokenIterator + + tokenIterator = TokenIterator(model_response, acompletion=acompletion) + response = CustomStreamWrapper( + completion_stream=tokenIterator, + model=model, + custom_llm_provider="sagemaker", + logging_obj=logging, + ) + ## LOGGING + logging.post_call( + input=messages, + api_key=None, + original_response=response, + ) + return response + + ## RESPONSE OBJECT + response = model_response + elif custom_llm_provider == "bedrock": + # boto3 reads keys from .env + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + response = bedrock.completion( + model=model, + messages=messages, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] == True + and not isinstance(response, CustomStreamWrapper) + ): + # don't try to access stream object, + if "ai21" in model: + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="bedrock", + logging_obj=logging, + ) + else: + response = CustomStreamWrapper( + iter(response), + model, + custom_llm_provider="bedrock", + logging_obj=logging, + ) + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=None, + original_response=response, + ) + + ## RESPONSE OBJECT + response = response + elif custom_llm_provider == "watsonx": + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + response = watsonx.IBMWatsonXAI().completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + timeout=timeout, # type: ignore + ) + if ( + "stream" in optional_params + and optional_params["stream"] == True + and not isinstance(response, CustomStreamWrapper) + ): + # don't try to access stream object, + response = CustomStreamWrapper( + iter(response), + model, + custom_llm_provider="watsonx", + logging_obj=logging, + ) + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=None, + original_response=response, + ) + ## RESPONSE OBJECT + response = response + elif custom_llm_provider == "vllm": + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = vllm.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + ) + + if ( + "stream" in optional_params and optional_params["stream"] == True + ): ## [BETA] + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vllm", + logging_obj=logging, + ) + return response + + ## RESPONSE OBJECT + response = model_response + elif custom_llm_provider == "ollama": + api_base = ( + litellm.api_base + or api_base + or get_secret("OLLAMA_API_BASE") + or "http://localhost:11434" + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + if model in custom_prompt_dict: + # check if the model has a registered custom prompt + model_prompt_details = custom_prompt_dict[model] + prompt = custom_prompt( + role_dict=model_prompt_details["roles"], + initial_prompt_value=model_prompt_details["initial_prompt_value"], + final_prompt_value=model_prompt_details["final_prompt_value"], + messages=messages, + ) + else: + prompt = prompt_factory( + model=model, + messages=messages, + custom_llm_provider=custom_llm_provider, + ) + if isinstance(prompt, dict): + # for multimode models - ollama/llava prompt_factory returns a dict { + # "prompt": prompt, + # "images": images + # } + prompt, images = prompt["prompt"], prompt["images"] + optional_params["images"] = images + + ## LOGGING + generator = ollama.get_ollama_response( + api_base, + model, + prompt, + optional_params, + logging_obj=logging, + acompletion=acompletion, + model_response=model_response, + encoding=encoding, + ) + if acompletion is True or optional_params.get("stream", False) == True: + return generator + + response = generator + elif custom_llm_provider == "ollama_chat": + api_base = ( + litellm.api_base + or api_base + or get_secret("OLLAMA_API_BASE") + or "http://localhost:11434" + ) + + api_key = ( + api_key + or litellm.ollama_key + or os.environ.get("OLLAMA_API_KEY") + or litellm.api_key + ) + ## LOGGING + generator = ollama_chat.get_ollama_response( + api_base, + api_key, + model, + messages, + optional_params, + logging_obj=logging, + acompletion=acompletion, + model_response=model_response, + encoding=encoding, + ) + if acompletion is True or optional_params.get("stream", False) == True: + return generator + + response = generator + elif custom_llm_provider == "cloudflare": + api_key = ( + api_key + or litellm.cloudflare_api_key + or litellm.api_key + or get_secret("CLOUDFLARE_API_KEY") + ) + account_id = get_secret("CLOUDFLARE_ACCOUNT_ID") + api_base = ( + api_base + or litellm.api_base + or get_secret("CLOUDFLARE_API_BASE") + or f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + response = cloudflare.completion( + model=model, + messages=messages, + api_base=api_base, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + ) + if "stream" in optional_params and optional_params["stream"] == True: + # don't try to access stream object, + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="cloudflare", + logging_obj=logging, + ) + + if optional_params.get("stream", False) or acompletion == True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + elif ( + custom_llm_provider == "baseten" + or litellm.api_base == "https://app.baseten.co" + ): + custom_llm_provider = "baseten" + baseten_key = ( + api_key + or litellm.baseten_key + or os.environ.get("BASETEN_API_KEY") + or litellm.api_key + ) + + model_response = baseten.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + api_key=baseten_key, + logging_obj=logging, + ) + if inspect.isgenerator(model_response) or ( + "stream" in optional_params and optional_params["stream"] == True + ): + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="baseten", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "petals" or model in litellm.petals_models: + api_base = api_base or litellm.api_base + + custom_llm_provider = "petals" + stream = optional_params.pop("stream", False) + model_response = petals.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + ) + if stream == True: ## [BETA] + # Fake streaming for petals + resp_string = model_response["choices"][0]["message"]["content"] + response = CustomStreamWrapper( + resp_string, + model, + custom_llm_provider="petals", + logging_obj=logging, + ) + return response + response = model_response + elif custom_llm_provider == "custom": + import requests + + url = litellm.api_base or api_base or "" + if url == None or url == "": + raise ValueError( + "api_base not set. Set api_base or litellm.api_base for custom endpoints" + ) + + """ + assume input to custom LLM api bases follow this format: + resp = requests.post( + api_base, + json={ + 'model': 'meta-llama/Llama-2-13b-hf', # model name + 'params': { + 'prompt': ["The capital of France is P"], + 'max_tokens': 32, + 'temperature': 0.7, + 'top_p': 1.0, + 'top_k': 40, + } + } + ) + + """ + prompt = " ".join([message["content"] for message in messages]) # type: ignore + resp = requests.post( + url, + json={ + "model": model, + "params": { + "prompt": [prompt], + "max_tokens": max_tokens, + "temperature": temperature, + "top_p": top_p, + "top_k": kwargs.get("top_k", 40), + }, + }, + ) + response_json = resp.json() + """ + assume all responses from custom api_bases of this format: + { + 'data': [ + { + 'prompt': 'The capital of France is P', + 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], + 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], + 'message': 'ok' + } + ] + } + """ + string_response = response_json["data"][0]["output"][0] + ## RESPONSE OBJECT + model_response["choices"][0]["message"]["content"] = string_response + model_response["created"] = int(time.time()) + model_response["model"] = model + response = model_response + else: + raise ValueError( + f"Unable to map your input to a model. Check your input - {args}" + ) + return response + except Exception as e: + ## Map to OpenAI Exception +> raise exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=args, + extra_kwargs=kwargs, + ) + +../main.py:2287: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +model = 'llama-3-8b-instruct', original_exception = KeyError('stream') +custom_llm_provider = 'predibase' +completion_kwargs = {'acompletion': False, 'api_base': None, 'api_key': 'pb_Qg9YbQo7UqqHdu0ozxN_aw', 'api_version': None, ...} +extra_kwargs = {'api_base': 'https://serving.app.predibase.com', 'litellm_call_id': 'cf0ea464-1b45-4473-8e55-6bf6809df7a7', 'litellm_logging_obj': , 'tenant_id': 'c4768f95'} + + def exception_type( + model, + original_exception, + custom_llm_provider, + completion_kwargs={}, + extra_kwargs={}, + ): + global user_logger_fn, liteDebuggerClient + exception_mapping_worked = False + if litellm.suppress_debug_info is False: + print() # noqa + print( # noqa + "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m" # noqa + ) # noqa + print( # noqa + "LiteLLM.Info: If you need to debug this error, use `litellm.set_verbose=True'." # noqa + ) # noqa + print() # noqa + try: + if model: + error_str = str(original_exception) + if isinstance(original_exception, BaseException): + exception_type = type(original_exception).__name__ + else: + exception_type = "" + + ################################################################################ + # Common Extra information needed for all providers + # We pass num retries, api_base, vertex_deployment etc to the exception here + ################################################################################ + + _api_base = litellm.get_api_base(model=model, optional_params=extra_kwargs) + messages = litellm.get_first_chars_messages(kwargs=completion_kwargs) + _vertex_project = extra_kwargs.get("vertex_project") + _vertex_location = extra_kwargs.get("vertex_location") + _metadata = extra_kwargs.get("metadata", {}) or {} + _model_group = _metadata.get("model_group") + _deployment = _metadata.get("deployment") + extra_information = f"\nModel: {model}" + if _api_base: + extra_information += f"\nAPI Base: {_api_base}" + if messages and len(messages) > 0: + extra_information += f"\nMessages: {messages}" + + if _model_group is not None: + extra_information += f"\nmodel_group: {_model_group}\n" + if _deployment is not None: + extra_information += f"\ndeployment: {_deployment}\n" + if _vertex_project is not None: + extra_information += f"\nvertex_project: {_vertex_project}\n" + if _vertex_location is not None: + extra_information += f"\nvertex_location: {_vertex_location}\n" + + # on litellm proxy add key name + team to exceptions + extra_information = _add_key_name_and_team_to_alert( + request_info=extra_information, metadata=_metadata + ) + + ################################################################################ + # End of Common Extra information Needed for all providers + ################################################################################ + + ################################################################################ + #################### Start of Provider Exception mapping #################### + ################################################################################ + + if "Request Timeout Error" in error_str or "Request timed out" in error_str: + exception_mapping_worked = True + raise Timeout( + message=f"APITimeoutError - Request timed out. {extra_information} \n error_str: {error_str}", + model=model, + llm_provider=custom_llm_provider, + ) + + if ( + custom_llm_provider == "openai" + or custom_llm_provider == "text-completion-openai" + or custom_llm_provider == "custom_openai" + or custom_llm_provider in litellm.openai_compatible_providers + ): + # custom_llm_provider is openai, make it OpenAI + if hasattr(original_exception, "message"): + message = original_exception.message + else: + message = str(original_exception) + if message is not None and isinstance(message, str): + message = message.replace("OPENAI", custom_llm_provider.upper()) + message = message.replace("openai", custom_llm_provider) + message = message.replace("OpenAI", custom_llm_provider) + if custom_llm_provider == "openai": + exception_provider = "OpenAI" + "Exception" + else: + exception_provider = ( + custom_llm_provider[0].upper() + + custom_llm_provider[1:] + + "Exception" + ) + + if ( + "This model's maximum context length is" in error_str + or "Request too large" in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "model_not_found" in error_str + ): + exception_mapping_worked = True + raise NotFoundError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "content_policy_violation" in error_str + ): + exception_mapping_worked = True + raise ContentPolicyViolationError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "Incorrect API key provided" not in error_str + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif "Mistral API raised a streaming error" in error_str: + exception_mapping_worked = True + _request = httpx.Request( + method="POST", url="https://api.openai.com/v1" + ) + raise APIError( + status_code=500, + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + request=_request, + ) + elif hasattr(original_exception, "status_code"): + exception_mapping_worked = True + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 404: + exception_mapping_worked = True + raise NotFoundError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + ) + elif original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 503: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 504: # gateway timeout error + exception_mapping_worked = True + raise Timeout( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + request=original_exception.request, + ) + else: + # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors + raise APIConnectionError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + request=httpx.Request( + method="POST", url="https://api.openai.com/v1/" + ), + ) + elif custom_llm_provider == "anthropic": # one of the anthropics + if hasattr(original_exception, "message"): + if ( + "prompt is too long" in original_exception.message + or "prompt: length" in original_exception.message + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=original_exception.message, + model=model, + llm_provider="anthropic", + response=original_exception.response, + ) + if "Invalid API Key" in original_exception.message: + exception_mapping_worked = True + raise AuthenticationError( + message=original_exception.message, + model=model, + llm_provider="anthropic", + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + print_verbose(f"status_code: {original_exception.status_code}") + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AnthropicException - {original_exception.message}", + llm_provider="anthropic", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 400 + or original_exception.status_code == 413 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"AnthropicException - {original_exception.message}", + model=model, + llm_provider="anthropic", + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"AnthropicException - {original_exception.message}", + model=model, + llm_provider="anthropic", + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AnthropicException - {original_exception.message}", + llm_provider="anthropic", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"AnthropicException - {original_exception.message}. Handle with `litellm.APIError`.", + llm_provider="anthropic", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "replicate": + if "Incorrect authentication token" in error_str: + exception_mapping_worked = True + raise AuthenticationError( + message=f"ReplicateException - {error_str}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif "input is too long" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"ReplicateException - {error_str}", + model=model, + llm_provider="replicate", + response=original_exception.response, + ) + elif exception_type == "ModelError": + exception_mapping_worked = True + raise BadRequestError( + message=f"ReplicateException - {error_str}", + model=model, + llm_provider="replicate", + response=original_exception.response, + ) + elif "Request was throttled" in error_str: + exception_mapping_worked = True + raise RateLimitError( + message=f"ReplicateException - {error_str}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"ReplicateException - {original_exception.message}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 400 + or original_exception.status_code == 422 + or original_exception.status_code == 413 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"ReplicateException - {original_exception.message}", + model=model, + llm_provider="replicate", + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"ReplicateException - {original_exception.message}", + model=model, + llm_provider="replicate", + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"ReplicateException - {original_exception.message}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"ReplicateException - {original_exception.message}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"ReplicateException - {str(original_exception)}", + llm_provider="replicate", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "bedrock": + if ( + "too many tokens" in error_str + or "expected maxLength:" in error_str + or "Input is too long" in error_str + or "prompt: length: 1.." in error_str + or "Too many input tokens" in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"BedrockException: Context Window Error - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if "Malformed input request" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"BedrockException - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if ( + "Unable to locate credentials" in error_str + or "The security token included in the request is invalid" + in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"BedrockException Invalid Authentication - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if "AccessDeniedException" in error_str: + exception_mapping_worked = True + raise PermissionDeniedError( + message=f"BedrockException PermissionDeniedError - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if ( + "throttlingException" in error_str + or "ThrottlingException" in error_str + ): + exception_mapping_worked = True + raise RateLimitError( + message=f"BedrockException: Rate Limit Error - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if "Connect timeout on endpoint URL" in error_str: + exception_mapping_worked = True + raise Timeout( + message=f"BedrockException: Timeout Error - {error_str}", + model=model, + llm_provider="bedrock", + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=httpx.Response( + status_code=500, + request=httpx.Request( + method="POST", url="https://api.openai.com/v1/" + ), + ), + ) + elif original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 404: + exception_mapping_worked = True + raise NotFoundError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=original_exception.response, + ) + elif custom_llm_provider == "sagemaker": + if "Unable to locate credentials" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"SagemakerException - {error_str}", + model=model, + llm_provider="sagemaker", + response=original_exception.response, + ) + elif ( + "Input validation error: `best_of` must be > 0 and <= 2" + in error_str + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"SagemakerException - the value of 'n' must be > 0 and <= 2 for sagemaker endpoints", + model=model, + llm_provider="sagemaker", + response=original_exception.response, + ) + elif ( + "`inputs` tokens + `max_new_tokens` must be <=" in error_str + or "instance type with more CPU capacity or memory" in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"SagemakerException - {error_str}", + model=model, + llm_provider="sagemaker", + response=original_exception.response, + ) + elif custom_llm_provider == "vertex_ai": + if ( + "Vertex AI API has not been used in project" in error_str + or "Unable to find your project" in error_str + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=original_exception.response, + ) + elif ( + "None Unknown Error." in error_str + or "Content has no parts." in error_str + ): + exception_mapping_worked = True + raise APIError( + message=f"VertexAIException - {error_str} {extra_information}", + status_code=500, + model=model, + llm_provider="vertex_ai", + request=original_exception.request, + ) + elif "403" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=original_exception.response, + ) + elif "The response was blocked." in error_str: + exception_mapping_worked = True + raise UnprocessableEntityError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ), + ) + elif ( + "429 Quota exceeded" in error_str + or "IndexError: list index out of range" in error_str + or "429 Unable to submit request because the service is temporarily out of capacity." + in error_str + ): + exception_mapping_worked = True + raise RateLimitError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ), + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=original_exception.response, + ) + if original_exception.status_code == 500: + exception_mapping_worked = True + raise APIError( + message=f"VertexAIException - {error_str} {extra_information}", + status_code=500, + model=model, + llm_provider="vertex_ai", + request=original_exception.request, + ) + elif custom_llm_provider == "palm" or custom_llm_provider == "gemini": + if "503 Getting metadata" in error_str: + # auth errors look like this + # 503 Getting metadata from plugin failed with error: Reauthentication is needed. Please run `gcloud auth application-default login` to reauthenticate. + exception_mapping_worked = True + raise BadRequestError( + message=f"GeminiException - Invalid api key", + model=model, + llm_provider="palm", + response=original_exception.response, + ) + if ( + "504 Deadline expired before operation could complete." in error_str + or "504 Deadline Exceeded" in error_str + ): + exception_mapping_worked = True + raise Timeout( + message=f"GeminiException - {original_exception.message}", + model=model, + llm_provider="palm", + ) + if "400 Request payload size exceeds" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"GeminiException - {error_str}", + model=model, + llm_provider="palm", + response=original_exception.response, + ) + if ( + "500 An internal error has occurred." in error_str + or "list index out of range" in error_str + ): + exception_mapping_worked = True + raise APIError( + status_code=getattr(original_exception, "status_code", 500), + message=f"GeminiException - {original_exception.message}", + llm_provider="palm", + model=model, + request=httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ), + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"GeminiException - {error_str}", + model=model, + llm_provider="palm", + response=original_exception.response, + ) + # Dailed: Error occurred: 400 Request payload size exceeds the limit: 20000 bytes + elif custom_llm_provider == "cloudflare": + if "Authentication error" in error_str: + exception_mapping_worked = True + raise AuthenticationError( + message=f"Cloudflare Exception - {original_exception.message}", + llm_provider="cloudflare", + model=model, + response=original_exception.response, + ) + if "must have required property" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"Cloudflare Exception - {original_exception.message}", + llm_provider="cloudflare", + model=model, + response=original_exception.response, + ) + elif ( + custom_llm_provider == "cohere" or custom_llm_provider == "cohere_chat" + ): # Cohere + if ( + "invalid api token" in error_str + or "No API key provided." in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif "too many tokens" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"CohereException - {original_exception.message}", + model=model, + llm_provider="cohere", + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + if ( + original_exception.status_code == 400 + or original_exception.status_code == 498 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif ( + "CohereConnectionError" in exception_type + ): # cohere seems to fire these errors when we load test it (1k+ messages / min) + exception_mapping_worked = True + raise RateLimitError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif "invalid type:" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif "Unexpected server error" in error_str: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + else: + if hasattr(original_exception, "status_code"): + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + request=original_exception.request, + ) + raise original_exception + elif custom_llm_provider == "huggingface": + if "length limit exceeded" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=error_str, + model=model, + llm_provider="huggingface", + response=original_exception.response, + ) + elif "A valid user token is required" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=error_str, + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"HuggingfaceException - {original_exception.message}", + model=model, + llm_provider="huggingface", + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"HuggingfaceException - {original_exception.message}", + model=model, + llm_provider="huggingface", + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 503: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "ai21": + if hasattr(original_exception, "message"): + if "Prompt has too many tokens" in original_exception.message: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + response=original_exception.response, + ) + if "Bad or missing API token." in original_exception.message: + exception_mapping_worked = True + raise BadRequestError( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AI21Exception - {original_exception.message}", + llm_provider="ai21", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + ) + if original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AI21Exception - {original_exception.message}", + llm_provider="ai21", + model=model, + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"AI21Exception - {original_exception.message}", + llm_provider="ai21", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "nlp_cloud": + if "detail" in error_str: + if "Input text length should not exceed" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"NLPCloudException - {error_str}", + model=model, + llm_provider="nlp_cloud", + response=original_exception.response, + ) + elif "value is not a valid" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"NLPCloudException - {error_str}", + model=model, + llm_provider="nlp_cloud", + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"NLPCloudException - {error_str}", + model=model, + llm_provider="nlp_cloud", + request=original_exception.request, + ) + if hasattr( + original_exception, "status_code" + ): # https://docs.nlpcloud.com/?shell#errors + if ( + original_exception.status_code == 400 + or original_exception.status_code == 406 + or original_exception.status_code == 413 + or original_exception.status_code == 422 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 401 + or original_exception.status_code == 403 + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 522 + or original_exception.status_code == 524 + ): + exception_mapping_worked = True + raise Timeout( + message=f"NLPCloudException - {original_exception.message}", + model=model, + llm_provider="nlp_cloud", + ) + elif ( + original_exception.status_code == 429 + or original_exception.status_code == 402 + ): + exception_mapping_worked = True + raise RateLimitError( + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 500 + or original_exception.status_code == 503 + ): + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + request=original_exception.request, + ) + elif ( + original_exception.status_code == 504 + or original_exception.status_code == 520 + ): + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"NLPCloudException - {original_exception.message}", + model=model, + llm_provider="nlp_cloud", + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "together_ai": + import json + + try: + error_response = json.loads(error_str) + except: + error_response = {"error": error_str} + if ( + "error" in error_response + and "`inputs` tokens + `max_new_tokens` must be <=" + in error_response["error"] + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + elif ( + "error" in error_response + and "invalid private key" in error_response["error"] + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"TogetherAIException - {error_response['error']}", + llm_provider="together_ai", + model=model, + response=original_exception.response, + ) + elif ( + "error" in error_response + and "INVALID_ARGUMENT" in error_response["error"] + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + + elif ( + "error" in error_response + and "API key doesn't match expected format." + in error_response["error"] + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + elif ( + "error_type" in error_response + and error_response["error_type"] == "validation" + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"TogetherAIException - {original_exception.message}", + model=model, + llm_provider="together_ai", + ) + elif original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"TogetherAIException - {original_exception.message}", + llm_provider="together_ai", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 524: + exception_mapping_worked = True + raise Timeout( + message=f"TogetherAIException - {original_exception.message}", + llm_provider="together_ai", + model=model, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"TogetherAIException - {original_exception.message}", + llm_provider="together_ai", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "aleph_alpha": + if ( + "This is longer than the model's maximum context length" + in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif "InvalidToken" in error_str or "No token provided" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + print_verbose(f"status code: {original_exception.status_code}") + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + ) + elif original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + raise original_exception + raise original_exception + elif ( + custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat" + ): + if isinstance(original_exception, dict): + error_str = original_exception.get("error", "") + else: + error_str = str(original_exception) + if "no such file or directory" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"OllamaException: Invalid Model/Model not loaded - {original_exception}", + model=model, + llm_provider="ollama", + response=original_exception.response, + ) + elif "Failed to establish a new connection" in error_str: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"OllamaException: {original_exception}", + llm_provider="ollama", + model=model, + response=original_exception.response, + ) + elif "Invalid response object from API" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"OllamaException: {original_exception}", + llm_provider="ollama", + model=model, + response=original_exception.response, + ) + elif "Read timed out" in error_str: + exception_mapping_worked = True + raise Timeout( + message=f"OllamaException: {original_exception}", + llm_provider="ollama", + model=model, + ) + elif custom_llm_provider == "vllm": + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 0: + exception_mapping_worked = True + raise APIConnectionError( + message=f"VLLMException - {original_exception.message}", + llm_provider="vllm", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "azure": + if "Internal server error" in error_str: + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + request=httpx.Request(method="POST", url="https://openai.com/"), + ) + elif "This model's maximum context length is" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif "DeploymentNotFound" in error_str: + exception_mapping_worked = True + raise NotFoundError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "content_policy_violation" in error_str + ) or ( + "The response was filtered due to the prompt triggering Azure OpenAI's content management" + in error_str + ): + exception_mapping_worked = True + raise ContentPolicyViolationError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif "invalid_request_error" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif ( + "The api_key client option must be set either by passing api_key to the client or by setting" + in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"{exception_provider} - {original_exception.message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + exception_mapping_worked = True + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + ) + if original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + response=original_exception.response, + ) + elif original_exception.status_code == 503: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + response=original_exception.response, + ) + elif original_exception.status_code == 504: # gateway timeout error + exception_mapping_worked = True + raise Timeout( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + request=httpx.Request( + method="POST", url="https://openai.com/" + ), + ) + else: + # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors + raise APIConnectionError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider="azure", + model=model, + request=httpx.Request(method="POST", url="https://openai.com/"), + ) + if ( + "BadRequestError.__init__() missing 1 required positional argument: 'param'" + in str(original_exception) + ): # deal with edge-case invalid request error bug in openai-python sdk + exception_mapping_worked = True + raise BadRequestError( + message=f"{exception_provider}: This can happen due to missing AZURE_API_VERSION: {str(original_exception)}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + else: # ensure generic errors always return APIConnectionError= + exception_mapping_worked = True + if hasattr(original_exception, "request"): + raise APIConnectionError( + message=f"{str(original_exception)}", + llm_provider=custom_llm_provider, + model=model, + request=original_exception.request, + ) + else: + raise APIConnectionError( + message=f"{str(original_exception)}", + llm_provider=custom_llm_provider, + model=model, + request=httpx.Request( + method="POST", url="https://api.openai.com/v1/" + ), # stub the request + ) + except Exception as e: + # LOGGING + exception_logging( + logger_fn=user_logger_fn, + additional_args={ + "exception_mapping_worked": exception_mapping_worked, + "original_exception": original_exception, + }, + exception=e, + ) + ## AUTH ERROR + if isinstance(e, AuthenticationError) and ( + litellm.email or "LITELLM_EMAIL" in os.environ + ): + threading.Thread(target=get_all_keys, args=(e.llm_provider,)).start() + # don't let an error with mapping interrupt the user from receiving an error from the llm api calls + if exception_mapping_worked: +> raise e + +../utils.py:9353: +_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ + +model = 'llama-3-8b-instruct', original_exception = KeyError('stream') +custom_llm_provider = 'predibase' +completion_kwargs = {'acompletion': False, 'api_base': None, 'api_key': 'pb_Qg9YbQo7UqqHdu0ozxN_aw', 'api_version': None, ...} +extra_kwargs = {'api_base': 'https://serving.app.predibase.com', 'litellm_call_id': 'cf0ea464-1b45-4473-8e55-6bf6809df7a7', 'litellm_logging_obj': , 'tenant_id': 'c4768f95'} + + def exception_type( + model, + original_exception, + custom_llm_provider, + completion_kwargs={}, + extra_kwargs={}, + ): + global user_logger_fn, liteDebuggerClient + exception_mapping_worked = False + if litellm.suppress_debug_info is False: + print() # noqa + print( # noqa + "\033[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new\033[0m" # noqa + ) # noqa + print( # noqa + "LiteLLM.Info: If you need to debug this error, use `litellm.set_verbose=True'." # noqa + ) # noqa + print() # noqa + try: + if model: + error_str = str(original_exception) + if isinstance(original_exception, BaseException): + exception_type = type(original_exception).__name__ + else: + exception_type = "" + + ################################################################################ + # Common Extra information needed for all providers + # We pass num retries, api_base, vertex_deployment etc to the exception here + ################################################################################ + + _api_base = litellm.get_api_base(model=model, optional_params=extra_kwargs) + messages = litellm.get_first_chars_messages(kwargs=completion_kwargs) + _vertex_project = extra_kwargs.get("vertex_project") + _vertex_location = extra_kwargs.get("vertex_location") + _metadata = extra_kwargs.get("metadata", {}) or {} + _model_group = _metadata.get("model_group") + _deployment = _metadata.get("deployment") + extra_information = f"\nModel: {model}" + if _api_base: + extra_information += f"\nAPI Base: {_api_base}" + if messages and len(messages) > 0: + extra_information += f"\nMessages: {messages}" + + if _model_group is not None: + extra_information += f"\nmodel_group: {_model_group}\n" + if _deployment is not None: + extra_information += f"\ndeployment: {_deployment}\n" + if _vertex_project is not None: + extra_information += f"\nvertex_project: {_vertex_project}\n" + if _vertex_location is not None: + extra_information += f"\nvertex_location: {_vertex_location}\n" + + # on litellm proxy add key name + team to exceptions + extra_information = _add_key_name_and_team_to_alert( + request_info=extra_information, metadata=_metadata + ) + + ################################################################################ + # End of Common Extra information Needed for all providers + ################################################################################ + + ################################################################################ + #################### Start of Provider Exception mapping #################### + ################################################################################ + + if "Request Timeout Error" in error_str or "Request timed out" in error_str: + exception_mapping_worked = True + raise Timeout( + message=f"APITimeoutError - Request timed out. {extra_information} \n error_str: {error_str}", + model=model, + llm_provider=custom_llm_provider, + ) + + if ( + custom_llm_provider == "openai" + or custom_llm_provider == "text-completion-openai" + or custom_llm_provider == "custom_openai" + or custom_llm_provider in litellm.openai_compatible_providers + ): + # custom_llm_provider is openai, make it OpenAI + if hasattr(original_exception, "message"): + message = original_exception.message + else: + message = str(original_exception) + if message is not None and isinstance(message, str): + message = message.replace("OPENAI", custom_llm_provider.upper()) + message = message.replace("openai", custom_llm_provider) + message = message.replace("OpenAI", custom_llm_provider) + if custom_llm_provider == "openai": + exception_provider = "OpenAI" + "Exception" + else: + exception_provider = ( + custom_llm_provider[0].upper() + + custom_llm_provider[1:] + + "Exception" + ) + + if ( + "This model's maximum context length is" in error_str + or "Request too large" in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "model_not_found" in error_str + ): + exception_mapping_worked = True + raise NotFoundError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "content_policy_violation" in error_str + ): + exception_mapping_worked = True + raise ContentPolicyViolationError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "Incorrect API key provided" not in error_str + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif ( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif "Mistral API raised a streaming error" in error_str: + exception_mapping_worked = True + _request = httpx.Request( + method="POST", url="https://api.openai.com/v1" + ) + raise APIError( + status_code=500, + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + request=_request, + ) + elif hasattr(original_exception, "status_code"): + exception_mapping_worked = True + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 404: + exception_mapping_worked = True + raise NotFoundError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + ) + elif original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 503: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + elif original_exception.status_code == 504: # gateway timeout error + exception_mapping_worked = True + raise Timeout( + message=f"{exception_provider} - {message} {extra_information}", + model=model, + llm_provider=custom_llm_provider, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + request=original_exception.request, + ) + else: + # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors + raise APIConnectionError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + request=httpx.Request( + method="POST", url="https://api.openai.com/v1/" + ), + ) + elif custom_llm_provider == "anthropic": # one of the anthropics + if hasattr(original_exception, "message"): + if ( + "prompt is too long" in original_exception.message + or "prompt: length" in original_exception.message + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=original_exception.message, + model=model, + llm_provider="anthropic", + response=original_exception.response, + ) + if "Invalid API Key" in original_exception.message: + exception_mapping_worked = True + raise AuthenticationError( + message=original_exception.message, + model=model, + llm_provider="anthropic", + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + print_verbose(f"status_code: {original_exception.status_code}") + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AnthropicException - {original_exception.message}", + llm_provider="anthropic", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 400 + or original_exception.status_code == 413 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"AnthropicException - {original_exception.message}", + model=model, + llm_provider="anthropic", + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"AnthropicException - {original_exception.message}", + model=model, + llm_provider="anthropic", + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AnthropicException - {original_exception.message}", + llm_provider="anthropic", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"AnthropicException - {original_exception.message}. Handle with `litellm.APIError`.", + llm_provider="anthropic", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "replicate": + if "Incorrect authentication token" in error_str: + exception_mapping_worked = True + raise AuthenticationError( + message=f"ReplicateException - {error_str}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif "input is too long" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"ReplicateException - {error_str}", + model=model, + llm_provider="replicate", + response=original_exception.response, + ) + elif exception_type == "ModelError": + exception_mapping_worked = True + raise BadRequestError( + message=f"ReplicateException - {error_str}", + model=model, + llm_provider="replicate", + response=original_exception.response, + ) + elif "Request was throttled" in error_str: + exception_mapping_worked = True + raise RateLimitError( + message=f"ReplicateException - {error_str}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"ReplicateException - {original_exception.message}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 400 + or original_exception.status_code == 422 + or original_exception.status_code == 413 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"ReplicateException - {original_exception.message}", + model=model, + llm_provider="replicate", + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"ReplicateException - {original_exception.message}", + model=model, + llm_provider="replicate", + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"ReplicateException - {original_exception.message}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"ReplicateException - {original_exception.message}", + llm_provider="replicate", + model=model, + response=original_exception.response, + ) + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"ReplicateException - {str(original_exception)}", + llm_provider="replicate", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "bedrock": + if ( + "too many tokens" in error_str + or "expected maxLength:" in error_str + or "Input is too long" in error_str + or "prompt: length: 1.." in error_str + or "Too many input tokens" in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"BedrockException: Context Window Error - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if "Malformed input request" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"BedrockException - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if ( + "Unable to locate credentials" in error_str + or "The security token included in the request is invalid" + in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"BedrockException Invalid Authentication - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if "AccessDeniedException" in error_str: + exception_mapping_worked = True + raise PermissionDeniedError( + message=f"BedrockException PermissionDeniedError - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if ( + "throttlingException" in error_str + or "ThrottlingException" in error_str + ): + exception_mapping_worked = True + raise RateLimitError( + message=f"BedrockException: Rate Limit Error - {error_str}", + model=model, + llm_provider="bedrock", + response=original_exception.response, + ) + if "Connect timeout on endpoint URL" in error_str: + exception_mapping_worked = True + raise Timeout( + message=f"BedrockException: Timeout Error - {error_str}", + model=model, + llm_provider="bedrock", + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=httpx.Response( + status_code=500, + request=httpx.Request( + method="POST", url="https://api.openai.com/v1/" + ), + ), + ) + elif original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 404: + exception_mapping_worked = True + raise NotFoundError( + message=f"BedrockException - {original_exception.message}", + llm_provider="bedrock", + model=model, + response=original_exception.response, + ) + elif custom_llm_provider == "sagemaker": + if "Unable to locate credentials" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"SagemakerException - {error_str}", + model=model, + llm_provider="sagemaker", + response=original_exception.response, + ) + elif ( + "Input validation error: `best_of` must be > 0 and <= 2" + in error_str + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"SagemakerException - the value of 'n' must be > 0 and <= 2 for sagemaker endpoints", + model=model, + llm_provider="sagemaker", + response=original_exception.response, + ) + elif ( + "`inputs` tokens + `max_new_tokens` must be <=" in error_str + or "instance type with more CPU capacity or memory" in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"SagemakerException - {error_str}", + model=model, + llm_provider="sagemaker", + response=original_exception.response, + ) + elif custom_llm_provider == "vertex_ai": + if ( + "Vertex AI API has not been used in project" in error_str + or "Unable to find your project" in error_str + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=original_exception.response, + ) + elif ( + "None Unknown Error." in error_str + or "Content has no parts." in error_str + ): + exception_mapping_worked = True + raise APIError( + message=f"VertexAIException - {error_str} {extra_information}", + status_code=500, + model=model, + llm_provider="vertex_ai", + request=original_exception.request, + ) + elif "403" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=original_exception.response, + ) + elif "The response was blocked." in error_str: + exception_mapping_worked = True + raise UnprocessableEntityError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ), + ) + elif ( + "429 Quota exceeded" in error_str + or "IndexError: list index out of range" in error_str + or "429 Unable to submit request because the service is temporarily out of capacity." + in error_str + ): + exception_mapping_worked = True + raise RateLimitError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ), + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"VertexAIException - {error_str} {extra_information}", + model=model, + llm_provider="vertex_ai", + response=original_exception.response, + ) + if original_exception.status_code == 500: + exception_mapping_worked = True + raise APIError( + message=f"VertexAIException - {error_str} {extra_information}", + status_code=500, + model=model, + llm_provider="vertex_ai", + request=original_exception.request, + ) + elif custom_llm_provider == "palm" or custom_llm_provider == "gemini": + if "503 Getting metadata" in error_str: + # auth errors look like this + # 503 Getting metadata from plugin failed with error: Reauthentication is needed. Please run `gcloud auth application-default login` to reauthenticate. + exception_mapping_worked = True + raise BadRequestError( + message=f"GeminiException - Invalid api key", + model=model, + llm_provider="palm", + response=original_exception.response, + ) + if ( + "504 Deadline expired before operation could complete." in error_str + or "504 Deadline Exceeded" in error_str + ): + exception_mapping_worked = True + raise Timeout( + message=f"GeminiException - {original_exception.message}", + model=model, + llm_provider="palm", + ) + if "400 Request payload size exceeds" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"GeminiException - {error_str}", + model=model, + llm_provider="palm", + response=original_exception.response, + ) + if ( + "500 An internal error has occurred." in error_str + or "list index out of range" in error_str + ): + exception_mapping_worked = True + raise APIError( + status_code=getattr(original_exception, "status_code", 500), + message=f"GeminiException - {original_exception.message}", + llm_provider="palm", + model=model, + request=httpx.Response( + status_code=429, + request=httpx.Request( + method="POST", + url=" https://cloud.google.com/vertex-ai/", + ), + ), + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"GeminiException - {error_str}", + model=model, + llm_provider="palm", + response=original_exception.response, + ) + # Dailed: Error occurred: 400 Request payload size exceeds the limit: 20000 bytes + elif custom_llm_provider == "cloudflare": + if "Authentication error" in error_str: + exception_mapping_worked = True + raise AuthenticationError( + message=f"Cloudflare Exception - {original_exception.message}", + llm_provider="cloudflare", + model=model, + response=original_exception.response, + ) + if "must have required property" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"Cloudflare Exception - {original_exception.message}", + llm_provider="cloudflare", + model=model, + response=original_exception.response, + ) + elif ( + custom_llm_provider == "cohere" or custom_llm_provider == "cohere_chat" + ): # Cohere + if ( + "invalid api token" in error_str + or "No API key provided." in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif "too many tokens" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"CohereException - {original_exception.message}", + model=model, + llm_provider="cohere", + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + if ( + original_exception.status_code == 400 + or original_exception.status_code == 498 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif ( + "CohereConnectionError" in exception_type + ): # cohere seems to fire these errors when we load test it (1k+ messages / min) + exception_mapping_worked = True + raise RateLimitError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif "invalid type:" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + elif "Unexpected server error" in error_str: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + response=original_exception.response, + ) + else: + if hasattr(original_exception, "status_code"): + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"CohereException - {original_exception.message}", + llm_provider="cohere", + model=model, + request=original_exception.request, + ) + raise original_exception + elif custom_llm_provider == "huggingface": + if "length limit exceeded" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=error_str, + model=model, + llm_provider="huggingface", + response=original_exception.response, + ) + elif "A valid user token is required" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=error_str, + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"HuggingfaceException - {original_exception.message}", + model=model, + llm_provider="huggingface", + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"HuggingfaceException - {original_exception.message}", + model=model, + llm_provider="huggingface", + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 503: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"HuggingfaceException - {original_exception.message}", + llm_provider="huggingface", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "ai21": + if hasattr(original_exception, "message"): + if "Prompt has too many tokens" in original_exception.message: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + response=original_exception.response, + ) + if "Bad or missing API token." in original_exception.message: + exception_mapping_worked = True + raise BadRequestError( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AI21Exception - {original_exception.message}", + llm_provider="ai21", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + ) + if original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"AI21Exception - {original_exception.message}", + model=model, + llm_provider="ai21", + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AI21Exception - {original_exception.message}", + llm_provider="ai21", + model=model, + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"AI21Exception - {original_exception.message}", + llm_provider="ai21", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "nlp_cloud": + if "detail" in error_str: + if "Input text length should not exceed" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"NLPCloudException - {error_str}", + model=model, + llm_provider="nlp_cloud", + response=original_exception.response, + ) + elif "value is not a valid" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"NLPCloudException - {error_str}", + model=model, + llm_provider="nlp_cloud", + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"NLPCloudException - {error_str}", + model=model, + llm_provider="nlp_cloud", + request=original_exception.request, + ) + if hasattr( + original_exception, "status_code" + ): # https://docs.nlpcloud.com/?shell#errors + if ( + original_exception.status_code == 400 + or original_exception.status_code == 406 + or original_exception.status_code == 413 + or original_exception.status_code == 422 + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 401 + or original_exception.status_code == 403 + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 522 + or original_exception.status_code == 524 + ): + exception_mapping_worked = True + raise Timeout( + message=f"NLPCloudException - {original_exception.message}", + model=model, + llm_provider="nlp_cloud", + ) + elif ( + original_exception.status_code == 429 + or original_exception.status_code == 402 + ): + exception_mapping_worked = True + raise RateLimitError( + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + response=original_exception.response, + ) + elif ( + original_exception.status_code == 500 + or original_exception.status_code == 503 + ): + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + request=original_exception.request, + ) + elif ( + original_exception.status_code == 504 + or original_exception.status_code == 520 + ): + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"NLPCloudException - {original_exception.message}", + model=model, + llm_provider="nlp_cloud", + response=original_exception.response, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"NLPCloudException - {original_exception.message}", + llm_provider="nlp_cloud", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "together_ai": + import json + + try: + error_response = json.loads(error_str) + except: + error_response = {"error": error_str} + if ( + "error" in error_response + and "`inputs` tokens + `max_new_tokens` must be <=" + in error_response["error"] + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + elif ( + "error" in error_response + and "invalid private key" in error_response["error"] + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"TogetherAIException - {error_response['error']}", + llm_provider="together_ai", + model=model, + response=original_exception.response, + ) + elif ( + "error" in error_response + and "INVALID_ARGUMENT" in error_response["error"] + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + + elif ( + "error" in error_response + and "API key doesn't match expected format." + in error_response["error"] + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + elif ( + "error_type" in error_response + and error_response["error_type"] == "validation" + ): + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"TogetherAIException - {original_exception.message}", + model=model, + llm_provider="together_ai", + ) + elif original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"TogetherAIException - {error_response['error']}", + model=model, + llm_provider="together_ai", + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"TogetherAIException - {original_exception.message}", + llm_provider="together_ai", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 524: + exception_mapping_worked = True + raise Timeout( + message=f"TogetherAIException - {original_exception.message}", + llm_provider="together_ai", + model=model, + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"TogetherAIException - {original_exception.message}", + llm_provider="together_ai", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "aleph_alpha": + if ( + "This is longer than the model's maximum context length" + in error_str + ): + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif "InvalidToken" in error_str or "No token provided" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + print_verbose(f"status code: {original_exception.status_code}") + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + ) + elif original_exception.status_code == 400: + exception_mapping_worked = True + raise BadRequestError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 500: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"AlephAlphaException - {original_exception.message}", + llm_provider="aleph_alpha", + model=model, + response=original_exception.response, + ) + raise original_exception + raise original_exception + elif ( + custom_llm_provider == "ollama" or custom_llm_provider == "ollama_chat" + ): + if isinstance(original_exception, dict): + error_str = original_exception.get("error", "") + else: + error_str = str(original_exception) + if "no such file or directory" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"OllamaException: Invalid Model/Model not loaded - {original_exception}", + model=model, + llm_provider="ollama", + response=original_exception.response, + ) + elif "Failed to establish a new connection" in error_str: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"OllamaException: {original_exception}", + llm_provider="ollama", + model=model, + response=original_exception.response, + ) + elif "Invalid response object from API" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"OllamaException: {original_exception}", + llm_provider="ollama", + model=model, + response=original_exception.response, + ) + elif "Read timed out" in error_str: + exception_mapping_worked = True + raise Timeout( + message=f"OllamaException: {original_exception}", + llm_provider="ollama", + model=model, + ) + elif custom_llm_provider == "vllm": + if hasattr(original_exception, "status_code"): + if original_exception.status_code == 0: + exception_mapping_worked = True + raise APIConnectionError( + message=f"VLLMException - {original_exception.message}", + llm_provider="vllm", + model=model, + request=original_exception.request, + ) + elif custom_llm_provider == "azure": + if "Internal server error" in error_str: + exception_mapping_worked = True + raise APIError( + status_code=500, + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + request=httpx.Request(method="POST", url="https://openai.com/"), + ) + elif "This model's maximum context length is" in error_str: + exception_mapping_worked = True + raise ContextWindowExceededError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif "DeploymentNotFound" in error_str: + exception_mapping_worked = True + raise NotFoundError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif ( + "invalid_request_error" in error_str + and "content_policy_violation" in error_str + ) or ( + "The response was filtered due to the prompt triggering Azure OpenAI's content management" + in error_str + ): + exception_mapping_worked = True + raise ContentPolicyViolationError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif "invalid_request_error" in error_str: + exception_mapping_worked = True + raise BadRequestError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif ( + "The api_key client option must be set either by passing api_key to the client or by setting" + in error_str + ): + exception_mapping_worked = True + raise AuthenticationError( + message=f"{exception_provider} - {original_exception.message} {extra_information}", + llm_provider=custom_llm_provider, + model=model, + response=original_exception.response, + ) + elif hasattr(original_exception, "status_code"): + exception_mapping_worked = True + if original_exception.status_code == 401: + exception_mapping_worked = True + raise AuthenticationError( + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) + elif original_exception.status_code == 408: + exception_mapping_worked = True + raise Timeout( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + ) + if original_exception.status_code == 422: + exception_mapping_worked = True + raise BadRequestError( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + response=original_exception.response, + ) + elif original_exception.status_code == 429: + exception_mapping_worked = True + raise RateLimitError( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + response=original_exception.response, + ) + elif original_exception.status_code == 503: + exception_mapping_worked = True + raise ServiceUnavailableError( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + response=original_exception.response, + ) + elif original_exception.status_code == 504: # gateway timeout error + exception_mapping_worked = True + raise Timeout( + message=f"AzureException - {original_exception.message} {extra_information}", + model=model, + llm_provider="azure", + ) + else: + exception_mapping_worked = True + raise APIError( + status_code=original_exception.status_code, + message=f"AzureException - {original_exception.message} {extra_information}", + llm_provider="azure", + model=model, + request=httpx.Request( + method="POST", url="https://openai.com/" + ), + ) + else: + # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors + raise APIConnectionError( + message=f"{exception_provider} - {message} {extra_information}", + llm_provider="azure", + model=model, + request=httpx.Request(method="POST", url="https://openai.com/"), + ) + if ( + "BadRequestError.__init__() missing 1 required positional argument: 'param'" + in str(original_exception) + ): # deal with edge-case invalid request error bug in openai-python sdk + exception_mapping_worked = True + raise BadRequestError( + message=f"{exception_provider}: This can happen due to missing AZURE_API_VERSION: {str(original_exception)}", + model=model, + llm_provider=custom_llm_provider, + response=original_exception.response, + ) + else: # ensure generic errors always return APIConnectionError= + exception_mapping_worked = True + if hasattr(original_exception, "request"): + raise APIConnectionError( + message=f"{str(original_exception)}", + llm_provider=custom_llm_provider, + model=model, + request=original_exception.request, + ) + else: +> raise APIConnectionError( + message=f"{str(original_exception)}", + llm_provider=custom_llm_provider, + model=model, + request=httpx.Request( + method="POST", url="https://api.openai.com/v1/" + ), # stub the request + ) +E litellm.exceptions.APIConnectionError: 'stream' + +../utils.py:9328: APIConnectionError + +During handling of the above exception, another exception occurred: + +sync_mode = True + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_completion_predibase_streaming(sync_mode): + try: + litellm.set_verbose = True + + if sync_mode: + response = completion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + stream=True, + ) + + complete_response = "" + for idx, init_chunk in enumerate(response): + chunk, finished = streaming_format_tests(idx, init_chunk) + complete_response += chunk + custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] + print(f"custom_llm_provider: {custom_llm_provider}") + assert custom_llm_provider == "predibase" + if finished: + assert isinstance( + init_chunk.choices[0], litellm.utils.StreamingChoices + ) + break + if complete_response.strip() == "": + raise Exception("Empty response received") + else: + response = await litellm.acompletion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + stream=True, + ) + + # await response + + complete_response = "" + idx = 0 + async for init_chunk in response: + chunk, finished = streaming_format_tests(idx, init_chunk) + complete_response += chunk + custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] + print(f"custom_llm_provider: {custom_llm_provider}") + assert custom_llm_provider == "predibase" + idx += 1 + if finished: + assert isinstance( + init_chunk.choices[0], litellm.utils.StreamingChoices + ) + break + if complete_response.strip() == "": + raise Exception("Empty response received") + + print(f"complete_response: {complete_response}") + except litellm.Timeout as e: + pass + except Exception as e: +> pytest.fail(f"Error occurred: {e}") +E Failed: Error occurred: 'stream' + +test_streaming.py:373: Failed +---------------------------- Captured stdout setup ----------------------------- + +----------------------------- Captured stdout call ----------------------------- + + +Request to litellm: +litellm.completion(model='predibase/llama-3-8b-instruct', tenant_id='c4768f95', api_base='https://serving.app.predibase.com', api_key='pb_Qg9YbQo7UqqHdu0ozxN_aw', messages=[{'role': 'user', 'content': 'What is the meaning of life?'}], stream=True) + + +self.optional_params: {} +SYNC kwargs[caching]: False; litellm.cache: None; kwargs.get('cache')['no-cache']: False +UNMAPPED PROVIDER, ASSUMING IT'S OPENAI/AZURE - model=llama-3-8b-instruct, custom_llm_provider=predibase +Final returned optional params: {'stream': True, 'tenant_id': 'c4768f95'} +self.optional_params: {'stream': True, 'tenant_id': 'c4768f95'} + + +POST Request Sent from LiteLLM: +curl -X POST \ +https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate_stream \ +-H 'content-type: application/json' -H 'Authorization: Bearer pb_Qg********************' \ +-d '{'inputs': 'What is the meaning of life?', 'parameters': {'details': True, 'max_new_tokens': 256, 'return_full_text': False}}' + + + +Give Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new +LiteLLM.Info: If you need to debug this error, use `litellm.set_verbose=True'. + +Logging Details: logger_fn - None | callable(logger_fn) - False +Logging Details LiteLLM-Failure Call +self.failure_callback: [] =============================== warnings summary =============================== -../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 24 warnings +../../../../../../opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: 25 warnings /opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_config.py:284: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ warnings.warn(DEPRECATION_MESSAGE, DeprecationWarning) @@ -47,86 +6906,90 @@ final cost: 0; prompt_tokens_cost_usd_dollar: 0; completion_tokens_cost_usd_doll /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:454: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @root_validator(pre=True) -../proxy/_types.py:466 - /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:466: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ +../proxy/_types.py:474 + /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:474: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @root_validator(pre=True) -../proxy/_types.py:509 - /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:509: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ +../proxy/_types.py:487 + /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:487: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @root_validator(pre=True) -../proxy/_types.py:546 - /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:546: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ +../proxy/_types.py:532 + /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:532: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @root_validator(pre=True) -../proxy/_types.py:840 - /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:840: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ +../proxy/_types.py:569 + /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:569: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @root_validator(pre=True) -../proxy/_types.py:867 - /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:867: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ +../proxy/_types.py:864 + /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:864: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @root_validator(pre=True) -../proxy/_types.py:886 - /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:886: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ +../proxy/_types.py:891 + /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:891: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ @root_validator(pre=True) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:121 - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:121: DeprecationWarning: pkg_resources is deprecated as an API - warnings.warn("pkg_resources is deprecated as an API", DeprecationWarning) +../proxy/_types.py:912 + /Users/krrishdholakia/Documents/litellm/litellm/proxy/_types.py:912: PydanticDeprecatedSince20: Pydantic V1 style `@root_validator` validators are deprecated. You should migrate to Pydantic V2 style `@model_validator` validators, see the migration guide for more details. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.7/migration/ + @root_validator(pre=True) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: 10 warnings - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google')`. +../utils.py:39 + /Users/krrishdholakia/Documents/litellm/litellm/utils.py:39: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html + import pkg_resources # type: ignore + +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: 10 warnings + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google.cloud')`. +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google.cloud')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2349 -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2349 -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2349 -test_completion.py::test_gemini_completion_call_error - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2349: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google')`. +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2317 +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2317 +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2317 + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2317: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(parent) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google.logging')`. +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google.logging')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google.iam')`. +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google.iam')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('mpl_toolkits')`. +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('mpl_toolkits')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`. +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('sphinxcontrib')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 -../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870 - /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2870: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('zope')`. +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 +../../../../../../opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832 + /opt/homebrew/lib/python3.11/site-packages/pkg_resources/__init__.py:2832: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('zope')`. Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages declare_namespace(pkg) -test_completion.py::test_gemini_completion_call_error - /opt/homebrew/lib/python3.11/site-packages/google/rpc/__init__.py:20: DeprecationWarning: Deprecated call to `pkg_resources.declare_namespace('google.rpc')`. - Implementing implicit namespace packages (as specified in PEP 420) is preferred to `pkg_resources.declare_namespace`. See https://setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages - pkg_resources.declare_namespace(__name__) +test_streaming.py::test_completion_predibase_streaming[False] + /opt/homebrew/lib/python3.11/site-packages/httpx/_content.py:204: DeprecationWarning: Use 'content=<...>' to upload raw bytes/text content. + warnings.warn(message, DeprecationWarning) -- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html -======================== 1 passed, 63 warnings in 1.48s ======================== +=========================== short test summary info ============================ +FAILED test_streaming.py::test_completion_predibase_streaming[True] - Failed:... +=================== 1 failed, 1 passed, 64 warnings in 5.28s =================== diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 7f0977b155c..f726ed95a7e 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -85,20 +85,33 @@ def test_completion_azure_command_r(): pytest.fail(f"Error occurred: {e}") -@pytest.mark.skip(reason="local test") -def test_completion_predibase(): +# @pytest.mark.skip(reason="local test") +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_completion_predibase(sync_mode): try: litellm.set_verbose = True - response = completion( - model="predibase/llama-3-8b-instruct", - tenant_id="c4768f95", - api_base="https://serving.app.predibase.com", - api_key=os.getenv("PREDIBASE_API_KEY"), - messages=[{"role": "user", "content": "What is the meaning of life?"}], - ) + if sync_mode: + response = completion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + ) - print(response) + print(response) + else: + response = await litellm.acompletion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + ) + + print(response) except litellm.Timeout as e: pass except Exception as e: diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index 7d639d7a3c4..7e5a1026514 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -5,6 +5,7 @@ import sys, os, asyncio import traceback import time, pytest from pydantic import BaseModel +from typing import Tuple sys.path.insert( 0, os.path.abspath("../..") @@ -142,7 +143,7 @@ def validate_last_format(chunk): ), "'finish_reason' should be a string." -def streaming_format_tests(idx, chunk): +def streaming_format_tests(idx, chunk) -> Tuple[str, bool]: extracted_chunk = "" finished = False print(f"chunk: {chunk}") @@ -306,6 +307,70 @@ def test_completion_azure_stream(): # test_completion_azure_stream() +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_completion_predibase_streaming(sync_mode): + try: + litellm.set_verbose = True + + if sync_mode: + response = completion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + stream=True, + ) + + complete_response = "" + for idx, init_chunk in enumerate(response): + chunk, finished = streaming_format_tests(idx, init_chunk) + complete_response += chunk + custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] + print(f"custom_llm_provider: {custom_llm_provider}") + assert custom_llm_provider == "predibase" + if finished: + assert isinstance( + init_chunk.choices[0], litellm.utils.StreamingChoices + ) + break + if complete_response.strip() == "": + raise Exception("Empty response received") + else: + response = await litellm.acompletion( + model="predibase/llama-3-8b-instruct", + tenant_id="c4768f95", + api_base="https://serving.app.predibase.com", + api_key=os.getenv("PREDIBASE_API_KEY"), + messages=[{"role": "user", "content": "What is the meaning of life?"}], + stream=True, + ) + + # await response + + complete_response = "" + idx = 0 + async for init_chunk in response: + chunk, finished = streaming_format_tests(idx, init_chunk) + complete_response += chunk + custom_llm_provider = init_chunk._hidden_params["custom_llm_provider"] + print(f"custom_llm_provider: {custom_llm_provider}") + assert custom_llm_provider == "predibase" + idx += 1 + if finished: + assert isinstance( + init_chunk.choices[0], litellm.utils.StreamingChoices + ) + break + if complete_response.strip() == "": + raise Exception("Empty response received") + + print(f"complete_response: {complete_response}") + except litellm.Timeout as e: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") def test_completion_azure_function_calling_stream(): diff --git a/litellm/utils.py b/litellm/utils.py index 7ccb5e8ff9f..fe52e6c6368 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9757,6 +9757,50 @@ class CustomStreamWrapper: "finish_reason": finish_reason, } + def handle_predibase_chunk(self, chunk): + try: + if type(chunk) != str: + chunk = chunk.decode( + "utf-8" + ) # DO NOT REMOVE this: This is required for HF inference API + Streaming + text = "" + is_finished = False + finish_reason = "" + print_verbose(f"chunk: {chunk}") + if chunk.startswith("data:"): + data_json = json.loads(chunk[5:]) + print_verbose(f"data json: {data_json}") + if "token" in data_json and "text" in data_json["token"]: + text = data_json["token"]["text"] + if data_json.get("details", False) and data_json["details"].get( + "finish_reason", False + ): + is_finished = True + finish_reason = data_json["details"]["finish_reason"] + elif data_json.get( + "generated_text", False + ): # if full generated text exists, then stream is complete + text = "" # don't return the final bos token + is_finished = True + finish_reason = "stop" + elif data_json.get("error", False): + raise Exception(data_json.get("error")) + return { + "text": text, + "is_finished": is_finished, + "finish_reason": finish_reason, + } + elif "error" in chunk: + raise ValueError(chunk) + return { + "text": text, + "is_finished": is_finished, + "finish_reason": finish_reason, + } + except Exception as e: + traceback.print_exc() + raise e + def handle_huggingface_chunk(self, chunk): try: if type(chunk) != str: @@ -10391,6 +10435,11 @@ class CustomStreamWrapper: completion_obj["content"] = response_obj["text"] if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] + elif self.custom_llm_provider and self.custom_llm_provider == "predibase": + response_obj = self.handle_predibase_chunk(chunk) + completion_obj["content"] = response_obj["text"] + if response_obj["is_finished"]: + self.received_finish_reason = response_obj["finish_reason"] elif ( self.custom_llm_provider and self.custom_llm_provider == "baseten" ): # baseten doesn't provide streaming @@ -11008,6 +11057,7 @@ class CustomStreamWrapper: or self.custom_llm_provider == "sagemaker" or self.custom_llm_provider == "gemini" or self.custom_llm_provider == "cached_response" + or self.custom_llm_provider == "predibase" or self.custom_llm_provider in litellm.openai_compatible_endpoints ): async for chunk in self.completion_stream: From 4a7be9163bb5dc4bbedf47c8b82b6bb18ee80bfb Mon Sep 17 00:00:00 2001 From: CyanideByte Date: Thu, 9 May 2024 17:42:19 -0700 Subject: [PATCH 044/169] Globally filtering pydantic conflict warnings --- litellm/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 4f72504f692..6f8d4e8dfac 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1,3 +1,6 @@ +### Hide pydantic namespace conflict warnings globally ### +import warnings +warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*") ### INIT VARIABLES ### import threading, requests, os from typing import Callable, List, Optional, Dict, Union, Any, Literal From 9083d8e490b485b9e72b75091b1a530833228ad0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 17:55:27 -0700 Subject: [PATCH 045/169] fix: fix linting errors --- .pre-commit-config.yaml | 16 ++++++++-------- litellm/llms/huggingface_restapi.py | 7 ++++--- litellm/llms/predibase.py | 8 ++++---- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e8bb1ff66a3..cc41d85f145 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,11 +16,11 @@ repos: name: Check if files match entry: python3 ci_cd/check_files_match.py language: system -# - repo: local -# hooks: -# - id: mypy -# name: mypy -# entry: python3 -m mypy --ignore-missing-imports -# language: system -# types: [python] -# files: ^litellm/ \ No newline at end of file +- repo: local + hooks: + - id: mypy + name: mypy + entry: python3 -m mypy --ignore-missing-imports + language: system + types: [python] + files: ^litellm/ \ No newline at end of file diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index b250f301383..a2c4457c23e 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -399,10 +399,11 @@ class Huggingface(BaseLLM): data = { "inputs": prompt, "parameters": optional_params, - "stream": ( + "stream": ( # type: ignore True if "stream" in optional_params - and optional_params["stream"] == True + and isinstance(optional_params["stream"], bool) + and optional_params["stream"] == True # type: ignore else False ), } @@ -433,7 +434,7 @@ class Huggingface(BaseLLM): data = { "inputs": prompt, "parameters": inference_params, - "stream": ( + "stream": ( # type: ignore True if "stream" in optional_params and optional_params["stream"] == True diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index f3935984d0d..ef9c6b0ba53 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -129,7 +129,7 @@ class PredibaseChatCompletion(BaseLLM): ) super().__init__() - def validate_environment(self, api_key: Optional[str], user_headers: dict) -> dict: + def _validate_environment(self, api_key: Optional[str], user_headers: dict) -> dict: if api_key is None: raise ValueError( "Missing Predibase API Key - A call is being made to predibase but no key is set either in the environment variables or via params" @@ -309,7 +309,7 @@ class PredibaseChatCompletion(BaseLLM): logger_fn=None, headers: dict = {}, ) -> Union[ModelResponse, CustomStreamWrapper]: - headers = self.validate_environment(api_key, headers) + headers = self._validate_environment(api_key, headers) completion_url = "" input_text = "" base_url = "https://serving.app.predibase.com" @@ -411,13 +411,13 @@ class PredibaseChatCompletion(BaseLLM): data=json.dumps(data), stream=stream, ) - response = CustomStreamWrapper( + _response = CustomStreamWrapper( response.iter_lines(), model, custom_llm_provider="predibase", logging_obj=logging_obj, ) - return response + return _response ### SYNC COMPLETION else: response = requests.post( From 425efc60f48421f7cbad2ce40fb4e6fabfc90a1b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 18:12:28 -0700 Subject: [PATCH 046/169] fix(main.py): fix linting error --- litellm/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index cec49d35f7e..273de4d2ee4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1810,7 +1810,7 @@ def completion( or get_secret("PREDIBASE_API_KEY") ) - model_response = predibase_chat_completions.completion( + _model_response = predibase_chat_completions.completion( model=model, messages=messages, model_response=model_response, @@ -1832,8 +1832,8 @@ def completion( and optional_params["stream"] == True and acompletion == False ): - return response - response = model_response + return _model_response + response = _model_response elif custom_llm_provider == "ai21": custom_llm_provider = "ai21" ai21_key = ( From 5a38438c3f32a6bdd9366fd0b42fe8673df26024 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 18:40:45 -0700 Subject: [PATCH 047/169] docs(customer_routing.md): add region-based routing for specific customers, to docs --- .../my-website/docs/proxy/customer_routing.md | 83 +++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 84 insertions(+) create mode 100644 docs/my-website/docs/proxy/customer_routing.md diff --git a/docs/my-website/docs/proxy/customer_routing.md b/docs/my-website/docs/proxy/customer_routing.md new file mode 100644 index 00000000000..4c8a60af807 --- /dev/null +++ b/docs/my-website/docs/proxy/customer_routing.md @@ -0,0 +1,83 @@ +# Region-based Routing + +Route specific customers to eu-only models. + +By specifying 'allowed_model_region' for a customer, LiteLLM will filter-out any models in a model group which is not in the allowed region (i.e. 'eu'). + +[**See Code**](https://github.com/BerriAI/litellm/blob/5eb12e30cc5faa73799ebc7e48fc86ebf449c879/litellm/router.py#L2938) + +### 1. Create customer with region-specification + +Use the litellm 'end-user' object for this. + +End-users can be tracked / id'ed by passing the 'user' param to litellm in an openai chat completion/embedding call. + +```bash +curl -X POST --location 'http://0.0.0.0:4000/end_user/new' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "user_id" : "ishaan-jaff-45", + "allowed_model_region": "eu", # 👈 SPECIFY ALLOWED REGION='eu' +}' +``` + +### 2. Add eu models to model-group + +Add eu models to a model group. For azure models, litellm can automatically infer the region (no need to set it). + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/gpt-35-turbo-eu # 👈 EU azure model + api_base: https://my-endpoint-europe-berri-992.openai.azure.com/ + api_key: os.environ/AZURE_EUROPE_API_KEY + - model_name: gpt-3.5-turbo + litellm_params: + model: azure/chatgpt-v-2 + api_base: https://openai-gpt-4-test-v-1.openai.azure.com/ + api_version: "2023-05-15" + api_key: os.environ/AZURE_API_KEY + +router_settings: + enable_pre_call_checks: true # 👈 IMPORTANT +``` + +Start the proxy + +```yaml +litellm --config /path/to/config.yaml +``` + +### 3. Test it! + +Make a simple chat completions call to the proxy. In the response headers, you should see the returned api base. + +```bash +curl -X POST --location 'http://localhost:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what is the meaning of the universe? 1234" + }], + "user": "ishaan-jaff-45" # 👈 USER ID +} +' +``` + +Expected API Base in response headers + +``` +x-litellm-api-base: "https://my-endpoint-europe-berri-992.openai.azure.com/" +``` + +### FAQ + +**What happens if there are no available models for that region?** + +Since the router filters out models not in the specified region, it will return back as an error to the user, if no models in that region are available. \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index d00d853a001..3c968ea57fd 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -50,6 +50,7 @@ const sidebars = { items: ["proxy/logging", "proxy/streaming_logging"], }, "proxy/team_based_routing", + "proxy/customer_routing", "proxy/ui", "proxy/cost_tracking", "proxy/token_auth", From 491e17734856fbd28260bbd3222b3ca9c49a86e0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 18:44:19 -0700 Subject: [PATCH 048/169] fix(predibase.py): fix async completion call --- litellm/llms/predibase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index ef9c6b0ba53..1f96e7c67e9 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -390,7 +390,7 @@ class PredibaseChatCompletion(BaseLLM): model=model, messages=messages, data=data, - api_base=api_base, + api_base=completion_url, model_response=model_response, print_verbose=print_verbose, encoding=encoding, From 76d4290591901961ac97cd53feee820b329b4dd5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 19:07:19 -0700 Subject: [PATCH 049/169] fix(predibase.py): fix event loop closed error --- litellm/llms/predibase.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index 1f96e7c67e9..e377b02d4ad 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -124,9 +124,6 @@ class PredibaseConfig: class PredibaseChatCompletion(BaseLLM): def __init__(self) -> None: - self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=litellm.request_timeout, connect=5.0) - ) super().__init__() def _validate_environment(self, api_key: Optional[str], user_headers: dict) -> dict: @@ -457,8 +454,10 @@ class PredibaseChatCompletion(BaseLLM): logger_fn=None, headers={}, ) -> ModelResponse: - - response = await self.async_handler.post( + async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) + response = await async_handler.post( api_base, headers=headers, data=json.dumps(data) ) return self.process_response( @@ -491,9 +490,11 @@ class PredibaseChatCompletion(BaseLLM): logger_fn=None, headers={}, ) -> CustomStreamWrapper: - + async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) data["stream"] = True - response = await self.async_handler.post( + response = await async_handler.post( url="https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate_stream", headers=headers, data=json.dumps(data), From 714370956fe74c503da238750d2a35df47beaeec Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 9 May 2024 22:18:16 -0700 Subject: [PATCH 050/169] fix(predibase.py): fix async streaming --- litellm/llms/predibase.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/litellm/llms/predibase.py b/litellm/llms/predibase.py index e377b02d4ad..c3424d244b3 100644 --- a/litellm/llms/predibase.py +++ b/litellm/llms/predibase.py @@ -454,10 +454,10 @@ class PredibaseChatCompletion(BaseLLM): logger_fn=None, headers={}, ) -> ModelResponse: - async_handler = AsyncHTTPHandler( + self.async_handler = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) - response = await async_handler.post( + response = await self.async_handler.post( api_base, headers=headers, data=json.dumps(data) ) return self.process_response( @@ -490,12 +490,12 @@ class PredibaseChatCompletion(BaseLLM): logger_fn=None, headers={}, ) -> CustomStreamWrapper: - async_handler = AsyncHTTPHandler( + self.async_handler = AsyncHTTPHandler( timeout=httpx.Timeout(timeout=600.0, connect=5.0) ) data["stream"] = True - response = await async_handler.post( - url="https://serving.app.predibase.com/c4768f95/deployments/v2/llms/llama-3-8b-instruct/generate_stream", + response = await self.async_handler.post( + url=api_base, headers=headers, data=json.dumps(data), stream=True, From 170fd11c8208f57739da1410d546bd6a9cb996ff Mon Sep 17 00:00:00 2001 From: Simon Sanchez Viloria Date: Fri, 10 May 2024 11:53:33 +0200 Subject: [PATCH 051/169] (fix) watsonx.py: Fixed linting errors and make sure stream chunk always return usage --- litellm/llms/watsonx.py | 365 ++++++++++++++++++++++++++++++---------- litellm/utils.py | 6 +- 2 files changed, 279 insertions(+), 92 deletions(-) diff --git a/litellm/llms/watsonx.py b/litellm/llms/watsonx.py index 082cdb3255b..ad4aff4b607 100644 --- a/litellm/llms/watsonx.py +++ b/litellm/llms/watsonx.py @@ -1,12 +1,25 @@ from enum import Enum import json, types, time # noqa: E401 from contextlib import asynccontextmanager, contextmanager -from typing import AsyncGenerator, Callable, Dict, Generator, Optional, Any, Union, List +from typing import ( + Callable, + Dict, + Generator, + AsyncGenerator, + Iterator, + AsyncIterator, + Optional, + Any, + Union, + List, + ContextManager, + AsyncContextManager, +) import httpx # type: ignore import requests # type: ignore import litellm -from litellm.utils import Logging, ModelResponse, Usage, get_secret +from litellm.utils import ModelResponse, Usage, get_secret from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from .base import BaseLLM @@ -189,6 +202,7 @@ class WatsonXAIEndpoint(str, Enum): ) EMBEDDINGS = "/ml/v1/text/embeddings" PROMPTS = "/ml/v1/prompts" + AVAILABLE_MODELS = "/ml/v1/foundation_model_specs" class IBMWatsonXAI(BaseLLM): @@ -378,12 +392,12 @@ class IBMWatsonXAI(BaseLLM): model_response: ModelResponse, print_verbose: Callable, encoding, - logging_obj: Logging, - optional_params: Optional[dict] = None, - acompletion: bool = None, - litellm_params: Optional[dict] = None, + logging_obj, + optional_params=None, + acompletion=None, + litellm_params=None, logger_fn=None, - timeout: Optional[float] = None, + timeout=None, ): """ Send a text generation request to the IBM Watsonx.ai API. @@ -403,8 +417,6 @@ class IBMWatsonXAI(BaseLLM): prompt = convert_messages_to_prompt( model, messages, provider, custom_prompt_dict ) - - manage_response = self._make_response_manager(async_=(acompletion is True), logging_obj=logging_obj) def process_text_gen_response(json_resp: dict) -> ModelResponse: if "results" not in json_resp: @@ -419,62 +431,72 @@ class IBMWatsonXAI(BaseLLM): model_response["finish_reason"] = json_resp["results"][0]["stop_reason"] model_response["created"] = int(time.time()) model_response["model"] = model - setattr( - model_response, - "usage", - Usage( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ), + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, ) + setattr(model_response, "usage", usage) return model_response + def process_stream_response( + stream_resp: Union[Iterator[str], AsyncIterator], + ) -> litellm.CustomStreamWrapper: + streamwrapper = litellm.CustomStreamWrapper( + stream_resp, + model=model, + custom_llm_provider="watsonx", + logging_obj=logging_obj, + ) + return streamwrapper + + # create the function to manage the request to watsonx.ai + # manage_request = self._make_request_manager( + # async_=(acompletion is True), logging_obj=logging_obj + # ) + self.request_manager = RequestManager(logging_obj) + def handle_text_request(request_params: dict) -> ModelResponse: - with manage_response( - request_params, input=prompt, timeout=timeout, + with self.request_manager.request( + request_params, + input=prompt, + timeout=timeout, ) as resp: json_resp = resp.json() return process_text_gen_response(json_resp) async def handle_text_request_async(request_params: dict) -> ModelResponse: - async with manage_response( - request_params, input=prompt, timeout=timeout, + async with self.request_manager.async_request( + request_params, + input=prompt, + timeout=timeout, ) as resp: json_resp = resp.json() return process_text_gen_response(json_resp) - def handle_stream_request( - request_params: dict, - ) -> litellm.CustomStreamWrapper: + def handle_stream_request(request_params: dict) -> litellm.CustomStreamWrapper: # stream the response - generated chunks will be handled # by litellm.utils.CustomStreamWrapper.handle_watsonx_stream - with manage_response( - request_params, stream=True, input=prompt, timeout=timeout, + with self.request_manager.request( + request_params, + stream=True, + input=prompt, + timeout=timeout, ) as resp: - streamwrapper = litellm.CustomStreamWrapper( - resp.iter_lines(), - model=model, - custom_llm_provider="watsonx", - logging_obj=logging_obj, - ) + streamwrapper = process_stream_response(resp.iter_lines()) return streamwrapper - async def handle_stream_request_async( - request_params: dict, - ) -> litellm.CustomStreamWrapper: + async def handle_stream_request_async(request_params: dict) -> litellm.CustomStreamWrapper: # stream the response - generated chunks will be handled # by litellm.utils.CustomStreamWrapper.handle_watsonx_stream - async with manage_response( - request_params, stream=True, input=prompt, timeout=timeout, + async with self.request_manager.async_request( + request_params, + stream=True, + input=prompt, + timeout=timeout, ) as resp: - streamwrapper = litellm.CustomStreamWrapper( - resp.aiter_lines(), - model=model, - custom_llm_provider="watsonx", - logging_obj=logging_obj, - ) + streamwrapper = process_stream_response(resp.aiter_lines()) return streamwrapper try: @@ -486,13 +508,13 @@ class IBMWatsonXAI(BaseLLM): optional_params=optional_params, print_verbose=print_verbose, ) - if stream and acompletion: + if stream and (acompletion is True): # stream and async text generation return handle_stream_request_async(req_params) elif stream: # streaming text generation return handle_stream_request(req_params) - elif acompletion: + elif (acompletion is True): # async text generation return handle_text_request_async(req_params) else: @@ -554,43 +576,48 @@ class IBMWatsonXAI(BaseLLM): "json": payload, "params": request_params, } - manage_response = self._make_response_manager(async_=(aembedding is True), logging_obj=logging_obj) - + # manage_request = self._make_request_manager( + # async_=(aembedding is True), logging_obj=logging_obj + # ) + request_manager = RequestManager(logging_obj) + def process_embedding_response(json_resp: dict) -> ModelResponse: results = json_resp.get("results", []) embedding_response = [] for idx, result in enumerate(results): embedding_response.append( - {"object": "embedding", "index": idx, "embedding": result["embedding"]} + { + "object": "embedding", + "index": idx, + "embedding": result["embedding"], + } ) model_response["object"] = "list" model_response["data"] = embedding_response model_response["model"] = model input_tokens = json_resp.get("input_token_count", 0) model_response.usage = Usage( - prompt_tokens=input_tokens, completion_tokens=0, total_tokens=input_tokens + prompt_tokens=input_tokens, + completion_tokens=0, + total_tokens=input_tokens, ) return model_response - - def handle_embedding_request(request_params: dict) -> ModelResponse: - with manage_response( - request_params, input=input - ) as resp: + + def handle_embedding(request_params: dict) -> ModelResponse: + with request_manager.request(request_params, input=input) as resp: json_resp = resp.json() return process_embedding_response(json_resp) - - async def handle_embedding_request_async(request_params: dict) -> ModelResponse: - async with manage_response( - request_params, input=input - ) as resp: + + async def handle_aembedding(request_params: dict) -> ModelResponse: + async with request_manager.async_request(request_params, input=input) as resp: json_resp = resp.json() return process_embedding_response(json_resp) - + try: - if aembedding: - return handle_embedding_request_async(req_params) + if aembedding is True: + return handle_embedding(req_params) else: - return handle_embedding_request(req_params) + return handle_aembedding(req_params) except WatsonXAIError as e: raise e except Exception as e: @@ -616,64 +643,88 @@ class IBMWatsonXAI(BaseLLM): iam_access_token = json_data["access_token"] self.token = iam_access_token return iam_access_token - - def _make_response_manager( - self, - async_: bool, - logging_obj: Logging - ) -> Callable[..., Generator[Union[requests.Response, httpx.Response], None, None]]: + + def get_available_models(self, *, ids_only: bool = True, **params): + api_params = self._get_api_params(params) + headers = { + "Authorization": f"Bearer {api_params['token']}", + "Content-Type": "application/json", + "Accept": "application/json", + } + request_params = dict(version=api_params["api_version"]) + url = api_params["url"].rstrip("/") + WatsonXAIEndpoint.AVAILABLE_MODELS + req_params = dict(method="GET", url=url, headers=headers, params=request_params) + # manage_request = self._make_request_manager(async_=False, logging_obj=None) + with RequestManager(logging_obj=None).request(req_params) as resp: + json_resp = resp.json() + if not ids_only: + return json_resp + return [res["model_id"] for res in json_resp["resources"]] + + def _make_request_manager( + self, async_: bool, logging_obj=None + ) -> Callable[ + ..., + Union[ContextManager[requests.Response], AsyncContextManager[httpx.Response]], + ]: """ Returns a context manager that manages the response from the request. if async_ is True, returns an async context manager, otherwise returns a regular context manager. Usage: ```python - manage_response = self._make_response_manager(async_=True, logging_obj=logging_obj) - async with manage_response(request_params) as resp: + manage_request = self._make_request_manager(async_=True, logging_obj=logging_obj) + async with manage_request(request_params) as resp: ... # or - manage_response = self._make_response_manager(async_=False, logging_obj=logging_obj) - with manage_response(request_params) as resp: + manage_request = self._make_request_manager(async_=False, logging_obj=logging_obj) + with manage_request(request_params) as resp: ... ``` """ def pre_call( request_params: dict, - input:Optional[Any]=None, + input: Optional[Any] = None, ): + if logging_obj is None: + return request_str = ( f"response = {'await ' if async_ else ''}{request_params['method']}(\n" f"\turl={request_params['url']},\n" - f"\tjson={request_params['json']},\n" + f"\tjson={request_params.get('json')},\n" f")" ) logging_obj.pre_call( input=input, api_key=request_params["headers"].get("Authorization"), additional_args={ - "complete_input_dict": request_params["json"], + "complete_input_dict": request_params.get("json"), "request_str": request_str, }, ) def post_call(resp, request_params): + if logging_obj is None: + return logging_obj.post_call( input=input, api_key=request_params["headers"].get("Authorization"), original_response=json.dumps(resp.json()), additional_args={ "status_code": resp.status_code, - "complete_input_dict": request_params.get("data", request_params.get("json")), + "complete_input_dict": request_params.get( + "data", request_params.get("json") + ), }, ) - + @contextmanager - def _manage_response( + def _manage_request( request_params: dict, stream: bool = False, input: Optional[Any] = None, - timeout: float = None, + timeout=None, ) -> Generator[requests.Response, None, None]: """ Returns a context manager that yields the response from the request. @@ -685,20 +736,23 @@ class IBMWatsonXAI(BaseLLM): request_params["stream"] = stream try: resp = requests.request(**request_params) - resp.raise_for_status() + if not resp.ok: + raise WatsonXAIError( + status_code=resp.status_code, + message=f"Error {resp.status_code} ({resp.reason}): {resp.text}", + ) yield resp except Exception as e: raise WatsonXAIError(status_code=500, message=str(e)) if not stream: post_call(resp, request_params) - - + @asynccontextmanager - async def _manage_response_async( + async def _manage_request_async( request_params: dict, stream: bool = False, input: Optional[Any] = None, - timeout: float = None, + timeout=None, ) -> AsyncGenerator[httpx.Response, None]: pre_call(request_params, input) if timeout: @@ -708,16 +762,23 @@ class IBMWatsonXAI(BaseLLM): try: # async with AsyncHTTPHandler(timeout=timeout) as client: self.async_handler = AsyncHTTPHandler( - timeout=httpx.Timeout(timeout=request_params.pop("timeout", 600.0), connect=5.0), + timeout=httpx.Timeout( + timeout=request_params.pop("timeout", 600.0), connect=5.0 + ), ) # async_handler.client.verify = False if "json" in request_params: - request_params['data'] = json.dumps(request_params.pop("json", {})) + request_params["data"] = json.dumps(request_params.pop("json", {})) method = request_params.pop("method") if method.upper() == "POST": resp = await self.async_handler.post(**request_params) else: resp = await self.async_handler.get(**request_params) + if resp.status_code not in [200, 201]: + raise WatsonXAIError( + status_code=resp.status_code, + message=f"Error {resp.status_code} ({resp.reason}): {resp.text}", + ) yield resp # await async_handler.close() except Exception as e: @@ -726,6 +787,132 @@ class IBMWatsonXAI(BaseLLM): post_call(resp, request_params) if async_: - return _manage_response_async + return _manage_request_async else: - return _manage_response + return _manage_request + +class RequestManager: + """ + Returns a context manager that manages the response from the request. + if async_ is True, returns an async context manager, otherwise returns a regular context manager. + + Usage: + ```python + request_params = dict(method="POST", url="https://api.example.com", headers={"Authorization" : "Bearer token"}, json={"key": "value"}) + request_manager = RequestManager(logging_obj=logging_obj) + async with request_manager.request(request_params) as resp: + ... + # or + with request_manager.async_request(request_params) as resp: + ... + ``` + """ + + def __init__(self, logging_obj=None): + self.logging_obj = logging_obj + + def pre_call( + self, + request_params: dict, + input: Optional[Any] = None, + ): + if self.logging_obj is None: + return + request_str = ( + f"response = {request_params['method']}(\n" + f"\turl={request_params['url']},\n" + f"\tjson={request_params.get('json')},\n" + f")" + ) + self.logging_obj.pre_call( + input=input, + api_key=request_params["headers"].get("Authorization"), + additional_args={ + "complete_input_dict": request_params.get("json"), + "request_str": request_str, + }, + ) + + def post_call(self, resp, request_params): + if self.logging_obj is None: + return + self.logging_obj.post_call( + input=input, + api_key=request_params["headers"].get("Authorization"), + original_response=json.dumps(resp.json()), + additional_args={ + "status_code": resp.status_code, + "complete_input_dict": request_params.get( + "data", request_params.get("json") + ), + }, + ) + + @contextmanager + def request( + self, + request_params: dict, + stream: bool = False, + input: Optional[Any] = None, + timeout=None, + ) -> Generator[requests.Response, None, None]: + """ + Returns a context manager that yields the response from the request. + """ + self.pre_call(request_params, input) + if timeout: + request_params["timeout"] = timeout + if stream: + request_params["stream"] = stream + try: + resp = requests.request(**request_params) + if not resp.ok: + raise WatsonXAIError( + status_code=resp.status_code, + message=f"Error {resp.status_code} ({resp.reason}): {resp.text}", + ) + yield resp + except Exception as e: + raise WatsonXAIError(status_code=500, message=str(e)) + if not stream: + self.post_call(resp, request_params) + + @asynccontextmanager + async def async_request( + self, + request_params: dict, + stream: bool = False, + input: Optional[Any] = None, + timeout=None, + ) -> AsyncGenerator[httpx.Response, None]: + self.pre_call(request_params, input) + if timeout: + request_params["timeout"] = timeout + if stream: + request_params["stream"] = stream + try: + # async with AsyncHTTPHandler(timeout=timeout) as client: + self.async_handler = AsyncHTTPHandler( + timeout=httpx.Timeout( + timeout=request_params.pop("timeout", 600.0), connect=5.0 + ), + ) + # async_handler.client.verify = False + if "json" in request_params: + request_params["data"] = json.dumps(request_params.pop("json", {})) + method = request_params.pop("method") + if method.upper() == "POST": + resp = await self.async_handler.post(**request_params) + else: + resp = await self.async_handler.get(**request_params) + if resp.status_code not in [200, 201]: + raise WatsonXAIError( + status_code=resp.status_code, + message=f"Error {resp.status_code} ({resp.reason}): {resp.text}", + ) + yield resp + # await async_handler.close() + except Exception as e: + raise WatsonXAIError(status_code=500, message=str(e)) + if not stream: + self.post_call(resp, request_params) \ No newline at end of file diff --git a/litellm/utils.py b/litellm/utils.py index d1af1b44a7b..3d1b0c1a709 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -10285,7 +10285,7 @@ class CustomStreamWrapper: response = chunk.replace("data: ", "").strip() parsed_response = json.loads(response) else: - return {"text": "", "is_finished": False} + return {"text": "", "is_finished": False, "prompt_tokens": 0, "completion_tokens": 0} else: print_verbose(f"chunk: {chunk} (Type: {type(chunk)})") raise ValueError( @@ -10300,8 +10300,8 @@ class CustomStreamWrapper: "text": text, "is_finished": is_finished, "finish_reason": finish_reason, - "prompt_tokens": results[0].get("input_token_count", None), - "completion_tokens": results[0].get("generated_token_count", None), + "prompt_tokens": results[0].get("input_token_count", 0), + "completion_tokens": results[0].get("generated_token_count", 0), } return {"text": "", "is_finished": False} except Exception as e: From d3d82827edbe9c5c3840795f1385c915629b957d Mon Sep 17 00:00:00 2001 From: Simon Sanchez Viloria Date: Fri, 10 May 2024 11:55:58 +0200 Subject: [PATCH 052/169] (test) Add tests for WatsonX completion/acompletion streaming --- litellm/tests/test_completion.py | 37 ++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 32b65faea5f..fa3e669f019 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -3089,6 +3089,24 @@ def test_completion_watsonx(): except Exception as e: pytest.fail(f"Error occurred: {e}") +def test_completion_stream_watsonx(): + litellm.set_verbose = True + model_name = "watsonx/ibm/granite-13b-chat-v2" + try: + response = completion( + model=model_name, + messages=messages, + stop=["stop"], + max_tokens=20, + stream=True + ) + for chunk in response: + print(chunk) + except litellm.APIError as e: + pass + except Exception as e: + pytest.fail(f"Error occurred: {e}") + @pytest.mark.parametrize( "provider, model, project, region_name, token", @@ -3153,6 +3171,25 @@ async def test_acompletion_watsonx(): except Exception as e: pytest.fail(f"Error occurred: {e}") +@pytest.mark.asyncio +async def test_acompletion_stream_watsonx(): + litellm.set_verbose = True + model_name = "watsonx/ibm/granite-13b-chat-v2" + print("testing watsonx") + try: + response = await litellm.acompletion( + model=model_name, + messages=messages, + temperature=0.2, + max_tokens=80, + stream=True + ) + # Add any assertions here to check the response + async for chunk in response: + print(chunk) + except Exception as e: + pytest.fail(f"Error occurred: {e}") + # test_completion_palm_stream() From 9a31f3d3d93b21eef8bcae45c0744a47c4c38ed9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 07:57:41 -0700 Subject: [PATCH 053/169] fix(main.py): support env var 'VERTEX_PROJECT' and 'VERTEX_LOCATION' --- litellm/main.py | 3 ++ .../tests/test_amazing_vertex_completion.py | 43 +++++++++++++++++++ litellm/utils.py | 4 +- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index aa078d322d0..6fd4cdaab48 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2960,17 +2960,20 @@ def embedding( or optional_params.pop("vertex_ai_project", None) or litellm.vertex_project or get_secret("VERTEXAI_PROJECT") + or get_secret("VERTEX_PROJECT") ) vertex_ai_location = ( optional_params.pop("vertex_location", None) or optional_params.pop("vertex_ai_location", None) or litellm.vertex_location or get_secret("VERTEXAI_LOCATION") + or get_secret("VERTEX_LOCATION") ) vertex_credentials = ( optional_params.pop("vertex_credentials", None) or optional_params.pop("vertex_ai_credentials", None) or get_secret("VERTEXAI_CREDENTIALS") + or get_secret("VERTEX_CREDENTIALS") ) response = vertex_ai.embedding( diff --git a/litellm/tests/test_amazing_vertex_completion.py b/litellm/tests/test_amazing_vertex_completion.py index 1d79653ea62..91fd444742b 100644 --- a/litellm/tests/test_amazing_vertex_completion.py +++ b/litellm/tests/test_amazing_vertex_completion.py @@ -113,6 +113,49 @@ async def get_response(): ], ) return response + + except litellm.UnprocessableEntityError as e: + pass + except Exception as e: + pytest.fail(f"An error occurred - {str(e)}") + + +@pytest.mark.asyncio +async def test_get_router_response(): + model = "claude-3-sonnet@20240229" + vertex_ai_project = "adroit-crow-413218" + vertex_ai_location = "asia-southeast1" + json_obj = get_vertex_ai_creds_json() + vertex_credentials = json.dumps(json_obj) + + prompt = '\ndef count_nums(arr):\n """\n Write a function count_nums which takes an array of integers and returns\n the number of elements which has a sum of digits > 0.\n If a number is negative, then its first signed digit will be negative:\n e.g. -123 has signed digits -1, 2, and 3.\n >>> count_nums([]) == 0\n >>> count_nums([-1, 11, -11]) == 1\n >>> count_nums([1, 1, 2]) == 3\n """\n' + try: + router = litellm.Router( + model_list=[ + { + "model_name": "sonnet", + "litellm_params": { + "model": "vertex_ai/claude-3-sonnet@20240229", + "vertex_ai_project": vertex_ai_project, + "vertex_ai_location": vertex_ai_location, + "vertex_credentials": vertex_credentials, + }, + } + ] + ) + response = await router.acompletion( + model="sonnet", + messages=[ + { + "role": "system", + "content": "Complete the given code with no more explanation. Remember that there is a 4-space indent before the first line of your generated code.", + }, + {"role": "user", "content": prompt}, + ], + ) + + print(f"\n\nResponse: {response}\n\n") + except litellm.UnprocessableEntityError as e: pass except Exception as e: diff --git a/litellm/utils.py b/litellm/utils.py index 206001dbb17..838d0fe553e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5769,9 +5769,7 @@ def get_optional_params( extra_body # openai client supports `extra_body` param ) else: # assume passing in params for openai/azure openai - print_verbose( - f"UNMAPPED PROVIDER, ASSUMING IT'S OPENAI/AZURE - model={model}, custom_llm_provider={custom_llm_provider}" - ) + supported_params = get_supported_openai_params( model=model, custom_llm_provider="openai" ) From f9a0364bffaa73b7cefb2d0375a9313b019cdf4e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 08:34:01 -0700 Subject: [PATCH 054/169] =?UTF-8?q?bump:=20version=201.37.0=20=E2=86=92=20?= =?UTF-8?q?1.37.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9854cf692a..835dfb3bf7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.37.0" +version = "1.37.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.37.0" +version = "1.37.1" version_files = [ "pyproject.toml:^version" ] From 40e19a838cdf123922d4cd861b4372ad493a843f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 08:40:31 -0700 Subject: [PATCH 055/169] =?UTF-8?q?bump:=20version=201.37.1=20=E2=86=92=20?= =?UTF-8?q?1.37.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 835dfb3bf7b..d09489e62d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.37.1" +version = "1.37.2" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.37.1" +version = "1.37.2" version_files = [ "pyproject.toml:^version" ] From cdec7a414f6376fbde82816e4ac4dece6d4c1e7c Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 09:58:40 -0700 Subject: [PATCH 056/169] test(test_router_fallbacks.py): fix test --- litellm/main.py | 1 + litellm/router.py | 1 + litellm/tests/test_custom_logger.py | 3 ++- litellm/tests/test_router_fallbacks.py | 20 +++++++++++--------- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 6fd4cdaab48..72f5b1dc63b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -662,6 +662,7 @@ def completion( "region_name", "allowed_model_region", ] + default_params = openai_params + litellm_params non_default_params = { k: v for k, v in kwargs.items() if k not in default_params diff --git a/litellm/router.py b/litellm/router.py index 68f49a0a0cb..39d49a14745 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -102,6 +102,7 @@ class Router: "usage-based-routing", "latency-based-routing", "cost-based-routing", + "usage-based-routing-v2", ] = "simple-shuffle", routing_strategy_args: dict = {}, # just for latency-based routing semaphore: Optional[asyncio.Semaphore] = None, diff --git a/litellm/tests/test_custom_logger.py b/litellm/tests/test_custom_logger.py index 9c2afe5a3a8..c7df3121462 100644 --- a/litellm/tests/test_custom_logger.py +++ b/litellm/tests/test_custom_logger.py @@ -437,8 +437,9 @@ async def test_cost_tracking_with_caching(): max_tokens=40, temperature=0.2, caching=True, + mock_response="Hey, i'm doing well!", ) - await asyncio.sleep(1) # success callback is async + await asyncio.sleep(3) # success callback is async response_cost = customHandler_optional_params.response_cost assert response_cost > 0 response2 = await litellm.acompletion( diff --git a/litellm/tests/test_router_fallbacks.py b/litellm/tests/test_router_fallbacks.py index 0e001eeba93..0bce9894b77 100644 --- a/litellm/tests/test_router_fallbacks.py +++ b/litellm/tests/test_router_fallbacks.py @@ -754,6 +754,9 @@ async def test_async_fallbacks_max_retries_per_request(): def test_ausage_based_routing_fallbacks(): try: + import litellm + + litellm.set_verbose = False # [Prod Test] # IT tests Usage Based Routing with fallbacks # The Request should fail azure/gpt-4-fast. Then fallback -> "azure/gpt-4-basic" -> "openai-gpt-4" @@ -766,10 +769,10 @@ def test_ausage_based_routing_fallbacks(): load_dotenv() # Constants for TPM and RPM allocation - AZURE_FAST_RPM = 0 - AZURE_BASIC_RPM = 0 + AZURE_FAST_RPM = 1 + AZURE_BASIC_RPM = 1 OPENAI_RPM = 0 - ANTHROPIC_RPM = 2 + ANTHROPIC_RPM = 10 def get_azure_params(deployment_name: str): params = { @@ -832,9 +835,9 @@ def test_ausage_based_routing_fallbacks(): fallbacks=fallbacks_list, set_verbose=True, debug_level="DEBUG", - routing_strategy="usage-based-routing", + routing_strategy="usage-based-routing-v2", redis_host=os.environ["REDIS_HOST"], - redis_port=os.environ["REDIS_PORT"], + redis_port=int(os.environ["REDIS_PORT"]), num_retries=0, ) @@ -853,8 +856,8 @@ def test_ausage_based_routing_fallbacks(): # the token count of this message is > AZURE_FAST_TPM, > AZURE_BASIC_TPM assert response._hidden_params["model_id"] == "1" - # now make 100 mock requests to OpenAI - expect it to fallback to anthropic-claude-instant-1.2 - for i in range(3): + for i in range(10): + # now make 100 mock requests to OpenAI - expect it to fallback to anthropic-claude-instant-1.2 response = router.completion( model="azure/gpt-4-fast", messages=messages, @@ -863,8 +866,7 @@ def test_ausage_based_routing_fallbacks(): ) print("response: ", response) print("response._hidden_params: ", response._hidden_params) - if i == 2: - # by the 19th call we should have hit TPM LIMIT for OpenAI, it should fallback to anthropic-claude-instant-1.2 + if i == 9: assert response._hidden_params["model_id"] == "4" except Exception as e: From 781d5888c39107058ae96966d114ce70cfe99581 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 10:58:35 -0700 Subject: [PATCH 057/169] docs(predibase.md): add support for predibase to docs --- docs/my-website/docs/providers/predibase.md | 247 ++++++++++++++++++++ docs/my-website/sidebars.js | 4 +- litellm/tests/test_completion.py | 1 - 3 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 docs/my-website/docs/providers/predibase.md diff --git a/docs/my-website/docs/providers/predibase.md b/docs/my-website/docs/providers/predibase.md new file mode 100644 index 00000000000..3d5bbaef417 --- /dev/null +++ b/docs/my-website/docs/providers/predibase.md @@ -0,0 +1,247 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 🆕 Predibase + +LiteLLM supports all models on Predibase + + +## Usage + + + + +### API KEYS +```python +import os +os.environ["PREDIBASE_API_KEY"] = "" +``` + +### Example Call + +```python +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" +os.environ["PREDIBASE_TENANT_ID"] = "predibase tenant id" + +# predibase llama-3 call +response = completion( + model="predibase/llama-3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + tenant_id: os.environ/PREDIBASE_TENANT_ID + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python + import openai + client = openai.OpenAI( + api_key="sk-1234", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="llama-3", + messages = [ + { + "role": "system", + "content": "Be a good human!" + }, + { + "role": "user", + "content": "What do you know about earth?" + } + ] + ) + + print(response) + ``` + + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "llama-3", + "messages": [ + { + "role": "system", + "content": "Be a good human!" + }, + { + "role": "user", + "content": "What do you know about earth?" + } + ], + }' + ``` + + + + + + + + + +## Advanced Usage - Prompt Formatting + +LiteLLM has prompt template mappings for all `meta-llama` llama3 instruct models. [**See Code**](https://github.com/BerriAI/litellm/blob/4f46b4c3975cd0f72b8c5acb2cb429d23580c18a/litellm/llms/prompt_templates/factory.py#L1360) + +To apply a custom prompt template: + + + + +```python +import litellm + +import os +os.environ["PREDIBASE_API_KEY"] = "" + +# Create your own custom prompt template +litellm.register_prompt_template( + model="togethercomputer/LLaMA-2-7B-32K", + initial_prompt_value="You are a good assistant" # [OPTIONAL] + roles={ + "system": { + "pre_message": "[INST] <>\n", # [OPTIONAL] + "post_message": "\n<>\n [/INST]\n" # [OPTIONAL] + }, + "user": { + "pre_message": "[INST] ", # [OPTIONAL] + "post_message": " [/INST]" # [OPTIONAL] + }, + "assistant": { + "pre_message": "\n" # [OPTIONAL] + "post_message": "\n" # [OPTIONAL] + } + } + final_prompt_value="Now answer as best you can:" # [OPTIONAL] +) + +def predibase_custom_model(): + model = "predibase/togethercomputer/LLaMA-2-7B-32K" + response = completion(model=model, messages=messages) + print(response['choices'][0]['message']['content']) + return response + +predibase_custom_model() +``` + + + +```yaml +# Model-specific parameters +model_list: + - model_name: mistral-7b # model alias + litellm_params: # actual params for litellm.completion() + model: "predibase/mistralai/Mistral-7B-Instruct-v0.1" + api_key: os.environ/PREDIBASE_API_KEY + initial_prompt_value: "\n" + roles: {"system":{"pre_message":"<|im_start|>system\n", "post_message":"<|im_end|>"}, "assistant":{"pre_message":"<|im_start|>assistant\n","post_message":"<|im_end|>"}, "user":{"pre_message":"<|im_start|>user\n","post_message":"<|im_end|>"}} + final_prompt_value: "\n" + bos_token: "" + eos_token: "" + max_tokens: 4096 +``` + + + + + +## Passing additional params - max_tokens, temperature +See all litellm.completion supported params [here](https://docs.litellm.ai/docs/completion/input) + +```python +# !pip install litellm +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" + +# predibae llama-3 call +response = completion( + model="predibase/llama3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}], + max_tokens=20, + temperature=0.5 +) +``` + +**proxy** + +```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + max_tokens: 20 + temperature: 0.5 +``` + +## Passings Predibase specific params - adapter_id, adapter_source, +Send params [not supported by `litellm.completion()`](https://docs.litellm.ai/docs/completion/input) but supported by Predibase by passing them to `litellm.completion` + +Example `adapter_id`, `adapter_source` are Predibase specific param - [See List](https://github.com/BerriAI/litellm/blob/8a35354dd6dbf4c2fcefcd6e877b980fcbd68c58/litellm/llms/predibase.py#L54) + +```python +# !pip install litellm +from litellm import completion +import os +## set ENV variables +os.environ["PREDIBASE_API_KEY"] = "predibase key" + +# predibase llama3 call +response = completion( + model="predibase/llama-3-8b-instruct", + messages = [{ "content": "Hello, how are you?","role": "user"}], + adapter_id="my_repo/3", + adapter_soruce="pbase", +) +``` + +**proxy** + +```yaml + model_list: + - model_name: llama-3 + litellm_params: + model: predibase/llama-3-8b-instruct + api_key: os.environ/PREDIBASE_API_KEY + adapter_id: my_repo/3 + adapter_source: pbase +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3c968ea57fd..4ce58708053 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -132,6 +132,8 @@ const sidebars = { "providers/cohere", "providers/anyscale", "providers/huggingface", + "providers/watsonx", + "providers/predibase", "providers/ollama", "providers/perplexity", "providers/groq", @@ -151,7 +153,7 @@ const sidebars = { "providers/openrouter", "providers/custom_openai_proxy", "providers/petals", - "providers/watsonx", + ], }, "proxy/custom_pricing", diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index f726ed95a7e..630baf34659 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -96,7 +96,6 @@ async def test_completion_predibase(sync_mode): response = completion( model="predibase/llama-3-8b-instruct", tenant_id="c4768f95", - api_base="https://serving.app.predibase.com", api_key=os.getenv("PREDIBASE_API_KEY"), messages=[{"role": "user", "content": "What is the meaning of life?"}], ) From 9d3f01c6ae23e63f20ff1f066b5ca12a9a29d5e1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:32:16 -0700 Subject: [PATCH 058/169] fix - router add model logic --- litellm/router.py | 40 ++++++++++++++++------------------------ 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 39d49a14745..5e71fea5837 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2558,20 +2558,20 @@ class Router: self.set_client(model=deployment.to_json(exclude_none=True)) # set region (if azure model) - try: - if "azure" in deployment.litellm_params.model: - region = litellm.utils.get_model_region( - litellm_params=deployment.litellm_params, mode=None - ) + # try: + # if "azure" in deployment.litellm_params.model: + # region = litellm.utils.get_model_region( + # litellm_params=deployment.litellm_params, mode=None + # ) - deployment.litellm_params.region_name = region - except Exception as e: - verbose_router_logger.error( - "Unable to get the region for azure model - {}, {}".format( - deployment.litellm_params.model, str(e) - ) - ) - pass # [NON-BLOCKING] + # deployment.litellm_params.region_name = region + # except Exception as e: + # verbose_router_logger.error( + # "Unable to get the region for azure model - {}, {}".format( + # deployment.litellm_params.model, str(e) + # ) + # ) + # pass # [NON-BLOCKING] return deployment @@ -2610,7 +2610,6 @@ class Router: - The added/updated deployment """ # check if deployment already exists - if deployment.model_info.id in self.get_model_ids(): # remove the previous deployment removal_idx: Optional[int] = None @@ -2620,16 +2619,9 @@ class Router: if removal_idx is not None: self.model_list.pop(removal_idx) - - # add to model list - _deployment = deployment.to_json(exclude_none=True) - self.model_list.append(_deployment) - - # initialize client - self._add_deployment(deployment=deployment) - - # add to model names - self.model_names.append(deployment.model_name) + else: + # if the model_id is not in router + self.add_deployment(deployment=deployment) return deployment def delete_deployment(self, id: str) -> Optional[Deployment]: From 6fd6490d6304043d9687723fb495a67e2749ac36 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:38:06 -0700 Subject: [PATCH 059/169] fix hide - _auto_infer_region behind a feature flag --- litellm/router.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5e71fea5837..3c777e2f07e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2558,20 +2558,23 @@ class Router: self.set_client(model=deployment.to_json(exclude_none=True)) # set region (if azure model) - # try: - # if "azure" in deployment.litellm_params.model: - # region = litellm.utils.get_model_region( - # litellm_params=deployment.litellm_params, mode=None - # ) + _auto_infer_region = os.environ.get("AUTO_INFER_REGION", "true") + _auto_infer_region_value = bool(_auto_infer_region) + if _auto_infer_region_value == True: + try: + if "azure" in deployment.litellm_params.model: + region = litellm.utils.get_model_region( + litellm_params=deployment.litellm_params, mode=None + ) - # deployment.litellm_params.region_name = region - # except Exception as e: - # verbose_router_logger.error( - # "Unable to get the region for azure model - {}, {}".format( - # deployment.litellm_params.model, str(e) - # ) - # ) - # pass # [NON-BLOCKING] + deployment.litellm_params.region_name = region + except Exception as e: + verbose_router_logger.error( + "Unable to get the region for azure model - {}, {}".format( + deployment.litellm_params.model, str(e) + ) + ) + pass # [NON-BLOCKING] return deployment From 75d6658bbcb54e9a9d73f38ece0a0f9d7be81866 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:39:19 -0700 Subject: [PATCH 060/169] fix - explain why behind feature flag --- litellm/router.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index 3c777e2f07e..ac9b4cb4794 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2561,6 +2561,10 @@ class Router: _auto_infer_region = os.environ.get("AUTO_INFER_REGION", "true") _auto_infer_region_value = bool(_auto_infer_region) if _auto_infer_region_value == True: + """ + Hiding behind a feature flag + When there is a large amount of LLM deployments this makes startup times blow up + """ try: if "azure" in deployment.litellm_params.model: region = litellm.utils.get_model_region( From 547976448f094af5875285cac400e3872446ee6a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 12:50:46 -0700 Subject: [PATCH 061/169] fix feature flag logic --- litellm/router.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac9b4cb4794..d833c1a85dd 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2558,9 +2558,8 @@ class Router: self.set_client(model=deployment.to_json(exclude_none=True)) # set region (if azure model) - _auto_infer_region = os.environ.get("AUTO_INFER_REGION", "true") - _auto_infer_region_value = bool(_auto_infer_region) - if _auto_infer_region_value == True: + _auto_infer_region = os.environ.get("AUTO_INFER_REGION", False) + if _auto_infer_region == True: """ Hiding behind a feature flag When there is a large amount of LLM deployments this makes startup times blow up From 5c69515a1347751f25e99671a1260cdc96108be8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:41:51 -0700 Subject: [PATCH 062/169] fix - upsert_deployment logic --- litellm/router.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index d833c1a85dd..20566f255bc 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2606,7 +2606,7 @@ class Router: self.model_names.append(deployment.model_name) return deployment - def upsert_deployment(self, deployment: Deployment) -> Deployment: + def upsert_deployment(self, deployment: Deployment) -> Deployment | None: """ Add or update deployment Parameters: @@ -2616,7 +2616,17 @@ class Router: - The added/updated deployment """ # check if deployment already exists - if deployment.model_info.id in self.get_model_ids(): + _deployment_model_id = deployment.model_info.id or "" + _deployment_on_router: Optional[Deployment] = self.get_deployment( + model_id=_deployment_model_id + ) + if _deployment_on_router is not None: + # deployment with this model_id exists on the router + if deployment.litellm_params == _deployment_on_router.litellm_params: + # No need to update + return None + + # if there is a new litellm param -> then update the deployment # remove the previous deployment removal_idx: Optional[int] = None for idx, model in enumerate(self.model_list): From 414af64343e696196d0ee3ddad3e1f51996ddb58 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:43:19 -0700 Subject: [PATCH 063/169] test - OpenAI client is re-used for Azure, OpenAI --- tests/test_models.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/test_models.py b/tests/test_models.py index b009ded6c11..9696c52a230 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -246,6 +246,33 @@ async def get_model_info_v2(session, key): raise Exception(f"Request did not return a 200 status code: {status}") +async def get_specific_model_info_v2(session, key, model_name): + url = "http://0.0.0.0:4000/v2/model/info?debug=True&model=" + model_name + print("running /model/info check for model=", model_name) + + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + } + + async with session.get(url, headers=headers) as response: + status = response.status + response_text = await response.text() + print("response from v2/model/info") + print(response_text) + print() + + _json_response = await response.json() + print("JSON response from /v2/model/info?model=", model_name, _json_response) + + _model_info = _json_response["data"] + assert len(_model_info) == 1, f"Expected 1 model, got {len(_model_info)}" + + if status != 200: + raise Exception(f"Request did not return a 200 status code: {status}") + return _model_info[0] + + async def get_model_health(session, key, model_name): url = "http://0.0.0.0:4000/health?model=" + model_name headers = { @@ -285,6 +312,11 @@ async def test_add_model_run_health(): model_name = f"azure-model-health-check-{model_id}" print("adding model", model_name) await add_model_for_health_checking(session=session, model_id=model_id) + _old_model_info = await get_specific_model_info_v2( + session=session, key=key, model_name=model_name + ) + print("model info before test", _old_model_info) + await asyncio.sleep(30) print("calling /model/info") await get_model_info(session=session, key=key) @@ -305,5 +337,28 @@ async def test_add_model_run_health(): _healthy_endpooint["model"] == "azure/chatgpt-v-2" ) # this is the model that got added + # assert httpx client is is unchanges + + await asyncio.sleep(10) + + _model_info_after_test = await get_specific_model_info_v2( + session=session, key=key, model_name=model_name + ) + + print("model info after test", _model_info_after_test) + old_openai_client = _old_model_info["openai_client"] + new_openai_client = _model_info_after_test["openai_client"] + print("old openai client", old_openai_client) + print("new openai client", new_openai_client) + + """ + PROD TEST - This is extremly important + The OpenAI client used should be the same after 30 seconds + It is a serious bug if the openai client does not match here + """ + assert ( + old_openai_client == new_openai_client + ), "OpenAI client does not match for the same model after 30 seconds" + # cleanup await delete_model(session=session, model_id=model_id) From 933f8ed16be60ff38223425b30a360c057995716 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:47:35 -0700 Subject: [PATCH 064/169] fix - proxy_server.py --- litellm/proxy/proxy_server.py | 38 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af1be7f2660..46c13277328 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -7795,11 +7795,15 @@ async def update_model( ) async def model_info_v2( user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), + model: Optional[str] = fastapi.Query( + None, description="Specify the model name (optional)" + ), + debug: Optional[bool] = False, ): """ BETA ENDPOINT. Might change unexpectedly. Use `/v1/model/info` for now. """ - global llm_model_list, general_settings, user_config_file_path, proxy_config + global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router if llm_model_list is None or not isinstance(llm_model_list, list): raise HTTPException( @@ -7822,19 +7826,35 @@ async def model_info_v2( if len(user_api_key_dict.models) > 0: user_models = user_api_key_dict.models + if model is not None: + all_models = [m for m in all_models if m["model_name"] == model] + # fill in model info based on config.yaml and litellm model_prices_and_context_window.json - for model in all_models: + for _model in all_models: # provided model_info in config.yaml - model_info = model.get("model_info", {}) + model_info = _model.get("model_info", {}) + if debug == True: + _openai_client = "None" + if llm_router is not None: + _openai_client = ( + llm_router._get_client( + deployment=_model, kwargs={}, client_type="async" + ) + or "None" + ) + else: + _openai_client = "llm_router_is_None" + openai_client = str(_openai_client) + _model["openai_client"] = openai_client # read litellm model_prices_and_context_window.json to get the following: # input_cost_per_token, output_cost_per_token, max_tokens - litellm_model_info = get_litellm_model_info(model=model) + litellm_model_info = get_litellm_model_info(model=_model) # 2nd pass on the model, try seeing if we can find model in litellm model_cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = _model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) try: litellm_model_info = litellm.get_model_info(model=litellm_model) @@ -7843,7 +7863,7 @@ async def model_info_v2( # 3rd pass on the model, try seeing if we can find model but without the "/" in model cost map if litellm_model_info == {}: # use litellm_param model_name to get model_info - litellm_params = model.get("litellm_params", {}) + litellm_params = _model.get("litellm_params", {}) litellm_model = litellm_params.get("model", None) split_model = litellm_model.split("/") if len(split_model) > 0: @@ -7855,10 +7875,10 @@ async def model_info_v2( for k, v in litellm_model_info.items(): if k not in model_info: model_info[k] = v - model["model_info"] = model_info + _model["model_info"] = model_info # don't return the api key / vertex credentials - model["litellm_params"].pop("api_key", None) - model["litellm_params"].pop("vertex_credentials", None) + _model["litellm_params"].pop("api_key", None) + _model["litellm_params"].pop("vertex_credentials", None) verbose_proxy_logger.debug("all_models: %s", all_models) return {"data": all_models} From 9bbb13c373313107a21d62e6e051d65b0afce91b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 13:54:52 -0700 Subject: [PATCH 065/169] fix bug upsert_deployment --- litellm/router.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/router.py b/litellm/router.py index 20566f255bc..32c2b61d13a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2606,7 +2606,7 @@ class Router: self.model_names.append(deployment.model_name) return deployment - def upsert_deployment(self, deployment: Deployment) -> Deployment | None: + def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]: """ Add or update deployment Parameters: From c17f221b8966e4b3581702822e2b8e181976a8f2 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:07:01 -0700 Subject: [PATCH 066/169] test(test_completion.py): reintegrate testing for huggingface tgi + non-tgi --- litellm/llms/huggingface_restapi.py | 103 +++++++++++++++++++--- litellm/tests/test_completion.py | 129 ++++++++++++++++++++++++++-- litellm/utils.py | 44 ++-------- 3 files changed, 218 insertions(+), 58 deletions(-) diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index a2c4457c23e..26591b95dac 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -6,10 +6,12 @@ import httpx, requests from .base import BaseLLM import time import litellm -from typing import Callable, Dict, List, Any +from typing import Callable, Dict, List, Any, Literal from litellm.utils import ModelResponse, Choices, Message, CustomStreamWrapper, Usage from typing import Optional from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm.types.completion import ChatCompletionMessageToolCallParam +import enum class HuggingfaceError(Exception): @@ -39,11 +41,29 @@ class HuggingfaceError(Exception): ) # Call the base class constructor with the parameters it needs +hf_task_list = [ + "text-generation-inference", + "conversational", + "text-classification", + "text-generation", +] + +hf_tasks = Literal[ + "text-generation-inference", + "conversational", + "text-classification", + "text-generation", +] + + class HuggingfaceConfig: """ Reference: https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/compat_generate """ + hf_task: Optional[hf_tasks] = ( + None # litellm-specific param, used to know the api spec to use when calling huggingface api + ) best_of: Optional[int] = None decoder_input_details: Optional[bool] = None details: Optional[bool] = True # enables returning logprobs + best of @@ -101,6 +121,51 @@ class HuggingfaceConfig: and v is not None } + def get_supported_openai_params(self): + return [ + "stream", + "temperature", + "max_tokens", + "top_p", + "stop", + "n", + "echo", + ] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + for param, value in non_default_params.items(): + # temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None + if param == "temperature": + if value == 0.0 or value == 0: + # hugging face exception raised when temp==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive + value = 0.01 + optional_params["temperature"] = value + if param == "top_p": + optional_params["top_p"] = value + if param == "n": + optional_params["best_of"] = value + optional_params["do_sample"] = ( + True # Need to sample if you want best of for hf inference endpoints + ) + if param == "stream": + optional_params["stream"] = value + if param == "stop": + optional_params["stop"] = value + if param == "max_tokens": + # HF TGI raises the following exception when max_new_tokens==0 + # Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive + if value == 0: + value = 1 + optional_params["max_new_tokens"] = value + if param == "echo": + # https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details + # Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False + optional_params["decoder_input_details"] = True + return optional_params + def output_parser(generated_text: str): """ @@ -162,7 +227,7 @@ def read_tgi_conv_models(): return set(), set() -def get_hf_task_for_model(model): +def get_hf_task_for_model(model: str) -> hf_tasks: # read text file, cast it to set # read the file called "huggingface_llms_metadata/hf_text_generation_models.txt" tgi_models, conversational_models = read_tgi_conv_models() @@ -171,7 +236,7 @@ def get_hf_task_for_model(model): elif model in conversational_models: return "conversational" elif "roneneldan/TinyStories" in model: - return None + return "text-generation" else: return "text-generation-inference" # default to tgi @@ -202,7 +267,7 @@ class Huggingface(BaseLLM): self, completion_response, model_response, - task, + task: hf_tasks, optional_params, encoding, input_text, @@ -270,6 +335,10 @@ class Huggingface(BaseLLM): ) choices_list.append(choice_obj) model_response["choices"].extend(choices_list) + elif task == "text-classification": + model_response["choices"][0]["message"]["content"] = json.dumps( + completion_response + ) else: if len(completion_response[0]["generated_text"]) > 0: model_response["choices"][0]["message"]["content"] = output_parser( @@ -332,7 +401,16 @@ class Huggingface(BaseLLM): exception_mapping_worked = False try: headers = self.validate_environment(api_key, headers) - task = get_hf_task_for_model(model) + if optional_params.get("hf_task") is None: + task = get_hf_task_for_model(model) + else: + task = optional_params.get("hf_task") # type: ignore + ## VALIDATE API FORMAT + if task is None or not isinstance(task, str) or task not in hf_task_list: + raise Exception( + "Invalid hf task - {}. Valid formats - {}.".format(task, hf_tasks) + ) + print_verbose(f"{model}, {task}") completion_url = "" input_text = "" @@ -433,14 +511,15 @@ class Huggingface(BaseLLM): inference_params.pop("return_full_text") data = { "inputs": prompt, - "parameters": inference_params, - "stream": ( # type: ignore + } + if task == "text-generation-inference": + data["parameters"] = inference_params + data["stream"] = ( # type: ignore True if "stream" in optional_params and optional_params["stream"] == True else False - ), - } + ) input_text = prompt ## LOGGING logging_obj.pre_call( @@ -531,10 +610,10 @@ class Huggingface(BaseLLM): isinstance(completion_response, dict) and "error" in completion_response ): - print_verbose(f"completion error: {completion_response['error']}") + print_verbose(f"completion error: {completion_response['error']}") # type: ignore print_verbose(f"response.status_code: {response.status_code}") raise HuggingfaceError( - message=completion_response["error"], + message=completion_response["error"], # type: ignore status_code=response.status_code, ) return self.convert_to_model_response_object( @@ -563,7 +642,7 @@ class Huggingface(BaseLLM): data: dict, headers: dict, model_response: ModelResponse, - task: str, + task: hf_tasks, encoding: Any, input_text: str, model: str, diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 630baf34659..4da489cc53d 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -13,6 +13,7 @@ import litellm from litellm import embedding, completion, completion_cost, Timeout from litellm import RateLimitError from litellm.llms.prompt_templates.factory import anthropic_messages_pt +from unittest.mock import patch, MagicMock # litellm.num_retries=3 litellm.cache = None @@ -1145,15 +1146,92 @@ def test_get_hf_task_for_model(): # ################### Hugging Face TGI models ######################## # # TGI model # # this is a TGI model https://huggingface.co/glaiveai/glaive-coder-7b -def hf_test_completion_tgi(): - # litellm.set_verbose=True +def tgi_mock_post(url, data=None, json=None, headers=None): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.json.return_value = [ + { + "generated_text": "<|assistant|>\nI'm", + "details": { + "finish_reason": "length", + "generated_tokens": 10, + "seed": None, + "prefill": [], + "tokens": [ + { + "id": 28789, + "text": "<", + "logprob": -0.025222778, + "special": False, + }, + { + "id": 28766, + "text": "|", + "logprob": -0.000003695488, + "special": False, + }, + { + "id": 489, + "text": "ass", + "logprob": -0.0000019073486, + "special": False, + }, + { + "id": 11143, + "text": "istant", + "logprob": -0.000002026558, + "special": False, + }, + { + "id": 28766, + "text": "|", + "logprob": -0.0000015497208, + "special": False, + }, + { + "id": 28767, + "text": ">", + "logprob": -0.0000011920929, + "special": False, + }, + { + "id": 13, + "text": "\n", + "logprob": -0.00009703636, + "special": False, + }, + {"id": 28737, "text": "I", "logprob": -0.1953125, "special": False}, + { + "id": 28742, + "text": "'", + "logprob": -0.88183594, + "special": False, + }, + { + "id": 28719, + "text": "m", + "logprob": -0.00032639503, + "special": False, + }, + ], + }, + } + ] + return mock_response + + +def test_hf_test_completion_tgi(): + litellm.set_verbose = True try: - response = completion( - model="huggingface/HuggingFaceH4/zephyr-7b-beta", - messages=[{"content": "Hello, how are you?", "role": "user"}], - ) - # Add any assertions here to check the response - print(response) + with patch("requests.post", side_effect=tgi_mock_post): + response = completion( + model="huggingface/HuggingFaceH4/zephyr-7b-beta", + messages=[{"content": "Hello, how are you?", "role": "user"}], + max_tokens=10, + ) + # Add any assertions here to check the response + print(response) except litellm.ServiceUnavailableError as e: pass except Exception as e: @@ -1191,6 +1269,41 @@ def hf_test_completion_tgi(): # except Exception as e: # pytest.fail(f"Error occurred: {e}") # hf_test_completion_none_task() + + +def mock_post(url, data=None, json=None, headers=None): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = {"Content-Type": "application/json"} + mock_response.json.return_value = [ + [ + {"label": "LABEL_0", "score": 0.9990691542625427}, + {"label": "LABEL_1", "score": 0.0009308889275416732}, + ] + ] + return mock_response + + +def test_hf_classifier_task(): + try: + with patch("requests.post", side_effect=mock_post): + litellm.set_verbose = True + user_message = "I like you. I love you" + messages = [{"content": user_message, "role": "user"}] + response = completion( + model="huggingface/shahrukhx01/question-vs-statement-classifier", + messages=messages, + hf_task="text-classification", + ) + print(f"response: {response}") + assert isinstance(response, litellm.ModelResponse) + assert isinstance(response.choices[0], litellm.Choices) + assert response.choices[0].message.content is not None + assert isinstance(response.choices[0].message.content, str) + except Exception as e: + pytest.fail(f"Error occurred: {str(e)}") + + ########################### End of Hugging Face Tests ############################################## # def test_completion_hf_api(): # # failing on circle ci commenting out diff --git a/litellm/utils.py b/litellm/utils.py index 838d0fe553e..f20cd220ce9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4840,6 +4840,7 @@ def get_optional_params_embeddings( def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create + model: str, functions=None, function_call=None, temperature=None, @@ -4853,7 +4854,6 @@ def get_optional_params( frequency_penalty=None, logit_bias=None, user=None, - model=None, custom_llm_provider="", response_format=None, seed=None, @@ -4882,7 +4882,7 @@ def get_optional_params( passed_params[k] = v - optional_params = {} + optional_params: Dict = {} common_auth_dict = litellm.common_cloud_provider_auth_params if custom_llm_provider in common_auth_dict["providers"]: @@ -5156,41 +5156,9 @@ def get_optional_params( model=model, custom_llm_provider=custom_llm_provider ) _check_valid_arg(supported_params=supported_params) - # temperature, top_p, n, stream, stop, max_tokens, n, presence_penalty default to None - if temperature is not None: - if temperature == 0.0 or temperature == 0: - # hugging face exception raised when temp==0 - # Failed: Error occurred: HuggingfaceException - Input validation error: `temperature` must be strictly positive - temperature = 0.01 - optional_params["temperature"] = temperature - if top_p is not None: - optional_params["top_p"] = top_p - if n is not None: - optional_params["best_of"] = n - optional_params["do_sample"] = ( - True # Need to sample if you want best of for hf inference endpoints - ) - if stream is not None: - optional_params["stream"] = stream - if stop is not None: - optional_params["stop"] = stop - if max_tokens is not None: - # HF TGI raises the following exception when max_new_tokens==0 - # Failed: Error occurred: HuggingfaceException - Input validation error: `max_new_tokens` must be strictly positive - if max_tokens == 0: - max_tokens = 1 - optional_params["max_new_tokens"] = max_tokens - if n is not None: - optional_params["best_of"] = n - if presence_penalty is not None: - optional_params["repetition_penalty"] = presence_penalty - if "echo" in passed_params: - # https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation.decoder_input_details - # Return the decoder input token logprobs and ids. You must set details=True as well for it to be taken into account. Defaults to False - optional_params["decoder_input_details"] = special_params["echo"] - passed_params.pop( - "echo", None - ) # since we handle translating echo, we should not send it to TGI request + optional_params = litellm.HuggingfaceConfig().map_openai_params( + non_default_params=non_default_params, optional_params=optional_params + ) elif custom_llm_provider == "together_ai": ## check if unsupported param passed in supported_params = get_supported_openai_params( @@ -6150,7 +6118,7 @@ def get_supported_openai_params(model: str, custom_llm_provider: str): "seed", ] elif custom_llm_provider == "huggingface": - return ["stream", "temperature", "max_tokens", "top_p", "stop", "n"] + return litellm.HuggingfaceConfig().get_supported_openai_params() elif custom_llm_provider == "together_ai": return [ "stream", From c744851d139265932123e81447261a174c14b47d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 10 May 2024 14:08:38 -0700 Subject: [PATCH 067/169] fix AUTO_INFER_REGION --- .circleci/config.yml | 1 + litellm/router.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ef8c0e33b7..08bbad6fac5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,6 +198,7 @@ jobs: -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ + -e AUTO_INFER_REGION="True" \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -e LANGFUSE_PROJECT1_PUBLIC=$LANGFUSE_PROJECT1_PUBLIC \ -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ diff --git a/litellm/router.py b/litellm/router.py index 32c2b61d13a..e8b0f658fa3 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2559,7 +2559,7 @@ class Router: # set region (if azure model) _auto_infer_region = os.environ.get("AUTO_INFER_REGION", False) - if _auto_infer_region == True: + if _auto_infer_region == True or _auto_infer_region == "True": """ Hiding behind a feature flag When there is a large amount of LLM deployments this makes startup times blow up From 50be25d11a05d7a4b15c43c35f35f969b678f324 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:08:47 -0700 Subject: [PATCH 068/169] test(test_optional_params.py): fix optional params --- litellm/tests/test_optional_params.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/tests/test_optional_params.py b/litellm/tests/test_optional_params.py index fac63ed67be..5c33cfa0e5c 100644 --- a/litellm/tests/test_optional_params.py +++ b/litellm/tests/test_optional_params.py @@ -86,6 +86,7 @@ def test_azure_optional_params_embeddings(): def test_azure_gpt_optional_params_gpt_vision(): # for OpenAI, Azure all extra params need to get passed as extra_body to OpenAI python. We assert we actually set extra_body here optional_params = litellm.utils.get_optional_params( + model="", user="John", custom_llm_provider="azure", max_tokens=10, @@ -125,6 +126,7 @@ def test_azure_gpt_optional_params_gpt_vision(): def test_azure_gpt_optional_params_gpt_vision_with_extra_body(): # if user passes extra_body, we should not over write it, we should pass it along to OpenAI python optional_params = litellm.utils.get_optional_params( + model="", user="John", custom_llm_provider="azure", max_tokens=10, @@ -167,6 +169,7 @@ def test_azure_gpt_optional_params_gpt_vision_with_extra_body(): def test_openai_extra_headers(): optional_params = litellm.utils.get_optional_params( + model="", user="John", custom_llm_provider="openai", max_tokens=10, From d4d175030fbf81afed5b16ff523c5e240e3dc1b6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:39:14 -0700 Subject: [PATCH 069/169] docs(huggingface.md): add text-classification to huggingface docs --- docs/my-website/docs/providers/huggingface.md | 176 +++++++++++++++++- litellm/llms/huggingface_restapi.py | 7 +- litellm/tests/test_completion.py | 3 +- 3 files changed, 177 insertions(+), 9 deletions(-) diff --git a/docs/my-website/docs/providers/huggingface.md b/docs/my-website/docs/providers/huggingface.md index f8ebadfcfa7..35befd3e205 100644 --- a/docs/my-website/docs/providers/huggingface.md +++ b/docs/my-website/docs/providers/huggingface.md @@ -21,6 +21,11 @@ This is done by adding the "huggingface/" prefix to `model`, example `completion +By default, LiteLLM will assume a huggingface call follows the TGI format. + + + + ```python import os from litellm import completion @@ -40,9 +45,58 @@ response = completion( print(response) ``` + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: wizard-coder + litellm_params: + model: huggingface/WizardLM/WizardCoder-Python-34B-V1.0 + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "wizard-coder", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + +Append `conversational` to the model name + +e.g. `huggingface/conversational/` + + + + ```python import os from litellm import completion @@ -54,7 +108,7 @@ messages = [{ "content": "There's a llama in my garden 😱 What should I do?"," # e.g. Call 'facebook/blenderbot-400M-distill' hosted on HF Inference endpoints response = completion( - model="huggingface/facebook/blenderbot-400M-distill", + model="huggingface/conversational/facebook/blenderbot-400M-distill", messages=messages, api_base="https://my-endpoint.huggingface.cloud" ) @@ -62,7 +116,123 @@ response = completion( print(response) ``` - + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: blenderbot + litellm_params: + model: huggingface/conversational/facebook/blenderbot-400M-distill + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "blenderbot", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + + + + +Append `text-classification` to the model name + +e.g. `huggingface/text-classification/` + + + + +```python +import os +from litellm import completion + +# [OPTIONAL] set env var +os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" + +messages = [{ "content": "I like you, I love you!","role": "user"}] + +# e.g. Call 'shahrukhx01/question-vs-statement-classifier' hosted on HF Inference endpoints +response = completion( + model="huggingface/text-classification/shahrukhx01/question-vs-statement-classifier", + messages=messages, + api_base="https://my-endpoint.endpoints.huggingface.cloud", +) + +print(response) +``` + + + +1. Add models to your config.yaml + + ```yaml + model_list: + - model_name: bert-classifier + litellm_params: + model: huggingface/text-classification/shahrukhx01/question-vs-statement-classifier + api_key: os.environ/HUGGINGFACE_API_KEY + api_base: "https://my-endpoint.endpoints.huggingface.cloud" + ``` + + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml --debug + ``` + +3. Test it! + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "bert-classifier", + "messages": [ + { + "role": "user", + "content": "I like you!" + } + ], + }' + ``` + + + + + + + +Append `text-generation` to the model name + +e.g. `huggingface/text-generation/` ```python import os @@ -75,7 +245,7 @@ messages = [{ "content": "There's a llama in my garden 😱 What should I do?"," # e.g. Call 'roneneldan/TinyStories-3M' hosted on HF Inference endpoints response = completion( - model="huggingface/roneneldan/TinyStories-3M", + model="huggingface/text-generation/roneneldan/TinyStories-3M", messages=messages, api_base="https://p69xlsj6rpno5drq.us-east-1.aws.endpoints.huggingface.cloud", ) diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 26591b95dac..ad3c570e762 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -230,6 +230,8 @@ def read_tgi_conv_models(): def get_hf_task_for_model(model: str) -> hf_tasks: # read text file, cast it to set # read the file called "huggingface_llms_metadata/hf_text_generation_models.txt" + if model.split("/")[0] in hf_task_list: + return model.split("/")[0] # type: ignore tgi_models, conversational_models = read_tgi_conv_models() if model in tgi_models: return "text-generation-inference" @@ -401,10 +403,7 @@ class Huggingface(BaseLLM): exception_mapping_worked = False try: headers = self.validate_environment(api_key, headers) - if optional_params.get("hf_task") is None: - task = get_hf_task_for_model(model) - else: - task = optional_params.get("hf_task") # type: ignore + task = get_hf_task_for_model(model) ## VALIDATE API FORMAT if task is None or not isinstance(task, str) or task not in hf_task_list: raise Exception( diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 4da489cc53d..b3e1928a8e5 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -1291,9 +1291,8 @@ def test_hf_classifier_task(): user_message = "I like you. I love you" messages = [{"content": user_message, "role": "user"}] response = completion( - model="huggingface/shahrukhx01/question-vs-statement-classifier", + model="huggingface/text-classification/shahrukhx01/question-vs-statement-classifier", messages=messages, - hf_task="text-classification", ) print(f"response: {response}") assert isinstance(response, litellm.ModelResponse) From 500995696ad2d1fe450d40461cc2f75ce2f24a1b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Fri, 10 May 2024 14:42:06 -0700 Subject: [PATCH 070/169] test: fix linting --- ...optional_params_functions_not_supported.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/litellm/tests/test_get_optional_params_functions_not_supported.py b/litellm/tests/test_get_optional_params_functions_not_supported.py index 2abfbc41f0a..c4580adba23 100644 --- a/litellm/tests/test_get_optional_params_functions_not_supported.py +++ b/litellm/tests/test_get_optional_params_functions_not_supported.py @@ -3,7 +3,27 @@ from litellm import get_optional_params litellm.add_function_to_prompt = True optional_params = get_optional_params( - tools= [{'type': 'function', 'function': {'description': 'Get the current weather in a given location', 'name': 'get_current_weather', 'parameters': {'type': 'object', 'properties': {'location': {'type': 'string', 'description': 'The city and state, e.g. San Francisco, CA'}, 'unit': {'type': 'string', 'enum': ['celsius', 'fahrenheit']}}, 'required': ['location']}}}], - tool_choice= 'auto', + model="", + tools=[ + { + "type": "function", + "function": { + "description": "Get the current weather in a given location", + "name": "get_current_weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ], + tool_choice="auto", ) -assert optional_params is not None \ No newline at end of file +assert optional_params is not None From 759ff3f750729f840b934bb3af09dd9844ce99f5 Mon Sep 17 00:00:00 2001 From: Nick Wong Date: Fri, 10 May 2024 15:42:07 -0700 Subject: [PATCH 071/169] added code to enforce unique key and team aliases in the ui --- .../src/components/create_key_button.tsx | 395 +++++++++------- ui/litellm-dashboard/src/components/teams.tsx | 439 +++++++++++------- 2 files changed, 495 insertions(+), 339 deletions(-) diff --git a/ui/litellm-dashboard/src/components/create_key_button.tsx b/ui/litellm-dashboard/src/components/create_key_button.tsx index 648eee9cadf..0e158778f96 100644 --- a/ui/litellm-dashboard/src/components/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/create_key_button.tsx @@ -2,8 +2,17 @@ import React, { useState, useEffect, useRef } from "react"; import { Button, TextInput, Grid, Col } from "@tremor/react"; -import { Card, Metric, Text, Title, Subtitle, Accordion, AccordionHeader, AccordionBody, } from "@tremor/react"; -import { CopyToClipboard } from 'react-copy-to-clipboard'; +import { + Card, + Metric, + Text, + Title, + Subtitle, + Accordion, + AccordionHeader, + AccordionBody, +} from "@tremor/react"; +import { CopyToClipboard } from "react-copy-to-clipboard"; import { Button as Button2, Modal, @@ -13,7 +22,11 @@ import { Select, message, } from "antd"; -import { keyCreateCall, slackBudgetAlertsHealthCheck, modelAvailableCall } from "./networking"; +import { + keyCreateCall, + slackBudgetAlertsHealthCheck, + modelAvailableCall, +} from "./networking"; const { Option } = Select; @@ -59,7 +72,11 @@ const CreateKey: React.FC = ({ } if (accessToken !== null) { - const model_available = await modelAvailableCall(accessToken, userID, userRole); + const model_available = await modelAvailableCall( + accessToken, + userID, + userRole + ); let available_model_names = model_available["data"].map( (element: { id: string }) => element.id ); @@ -70,12 +87,25 @@ const CreateKey: React.FC = ({ console.error("Error fetching user models:", error); } }; - + fetchUserModels(); }, [accessToken, userID, userRole]); const handleCreate = async (formValues: Record) => { try { + const newKeyAlias = formValues?.key_alias ?? ""; + const newKeyTeamId = formValues?.team_id ?? null; + const existingKeyAliases = + data + ?.filter((k) => k.team_id === newKeyTeamId) + .map((k) => k.key_alias) ?? []; + + if (existingKeyAliases.includes(newKeyAlias)) { + throw new Error( + `Key alias ${newKeyAlias} already exists for team with ID ${newKeyTeamId}, please provide another key alias` + ); + } + message.info("Making API Call"); setIsModalVisible(true); const response = await keyCreateCall(accessToken, userID, formValues); @@ -89,12 +119,13 @@ const CreateKey: React.FC = ({ localStorage.removeItem("userData" + userID); } catch (error) { console.error("Error creating the key:", error); + message.error(`Error creating the key: ${error}`, 20); } }; const handleCopy = () => { - message.success('API Key copied to clipboard'); -}; + message.success("API Key copied to clipboard"); + }; useEffect(() => { let tempModelsToPick = []; @@ -119,7 +150,6 @@ const CreateKey: React.FC = ({ setModelsToPick(tempModelsToPick); }, [team, userModels]); - return (
@@ -141,140 +171,164 @@ const CreateKey: React.FC = ({ wrapperCol={{ span: 16 }} labelAlign="left" > - <> - - - - - - - - - - - Optional Settings - - - { - if (value && team && team.max_budget !== null && value > team.max_budget) { - throw new Error(`Budget cannot exceed team max budget: $${team.max_budget}`); - } - }, - }, - ]} - > - - - + + + + - { - if (value && team && team.tpm_limit !== null && value > team.tpm_limit) { - throw new Error(`TPM limit cannot exceed team TPM limit: ${team.tpm_limit}`); - } - }, - }, - ]} - > - - - { - if (value && team && team.rpm_limit !== null && value > team.rpm_limit) { - throw new Error(`RPM limit cannot exceed team RPM limit: ${team.rpm_limit}`); - } - }, - }, - ]} - > - - - - - - - - + > + + + + + + + + + Optional Settings + + + { + if ( + value && + team && + team.max_budget !== null && + value > team.max_budget + ) { + throw new Error( + `Budget cannot exceed team max budget: $${team.max_budget}` + ); + } + }, + }, + ]} + > + + + + + + { + if ( + value && + team && + team.tpm_limit !== null && + value > team.tpm_limit + ) { + throw new Error( + `TPM limit cannot exceed team TPM limit: ${team.tpm_limit}` + ); + } + }, + }, + ]} + > + + + { + if ( + value && + team && + team.rpm_limit !== null && + value > team.rpm_limit + ) { + throw new Error( + `RPM limit cannot exceed team RPM limit: ${team.rpm_limit}` + ); + } + }, + }, + ]} + > + + + + + + + + + + + - - - -
Create Key
@@ -288,36 +342,45 @@ const CreateKey: React.FC = ({ footer={null} > - - Save your Key - -

- Please save this secret key somewhere safe and accessible. For - security reasons, you will not be able to view it again{" "} - through your LiteLLM account. If you lose this secret key, you - will need to generate a new one. -

- - - {apiKey != null ? ( -
+ Save your Key + +

+ Please save this secret key somewhere safe and accessible. For + security reasons, you will not be able to view it again{" "} + through your LiteLLM account. If you lose this secret key, you + will need to generate a new one. +

+ + + {apiKey != null ? ( +
API Key: -
- 
{apiKey}
-
- - +
+ 
+                      {apiKey}
+
+
+ + - - {/* */} -
- ) : ( - Key being created, this might take 30s - )} - - +
+ ) : ( + Key being created, this might take 30s + )} + )} diff --git a/ui/litellm-dashboard/src/components/teams.tsx b/ui/litellm-dashboard/src/components/teams.tsx index b2480179909..d22789420fd 100644 --- a/ui/litellm-dashboard/src/components/teams.tsx +++ b/ui/litellm-dashboard/src/components/teams.tsx @@ -2,7 +2,13 @@ import React, { useState, useEffect } from "react"; import Link from "next/link"; import { Typography } from "antd"; import { teamDeleteCall, teamUpdateCall, teamInfoCall } from "./networking"; -import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon } from "@heroicons/react/outline"; +import { + InformationCircleIcon, + PencilAltIcon, + PencilIcon, + StatusOnlineIcon, + TrashIcon, +} from "@heroicons/react/outline"; import { Button as Button2, Modal, @@ -46,8 +52,12 @@ interface EditTeamModalProps { onSubmit: (data: FormData) => void; // Assuming FormData is the type of data to be submitted } - -import { teamCreateCall, teamMemberAddCall, Member, modelAvailableCall } from "./networking"; +import { + teamCreateCall, + teamMemberAddCall, + Member, + modelAvailableCall, +} from "./networking"; const Team: React.FC = ({ teams, @@ -63,7 +73,6 @@ const Team: React.FC = ({ const [value, setValue] = useState(""); const [editModalVisible, setEditModalVisible] = useState(false); - const [selectedTeam, setSelectedTeam] = useState( teams ? teams[0] : null ); @@ -76,127 +85,125 @@ const Team: React.FC = ({ // store team info as {"team_id": team_info_object} const [perTeamInfo, setPerTeamInfo] = useState>({}); + const EditTeamModal: React.FC = ({ + visible, + onCancel, + team, + onSubmit, + }) => { + const [form] = Form.useForm(); - const EditTeamModal: React.FC = ({ visible, onCancel, team, onSubmit }) => { - const [form] = Form.useForm(); + const handleOk = () => { + form + .validateFields() + .then((values) => { + const updatedValues = { ...values, team_id: team.team_id }; + onSubmit(updatedValues); + form.resetFields(); + }) + .catch((error) => { + console.error("Validation failed:", error); + }); + }; - const handleOk = () => { - form - .validateFields() - .then((values) => { - const updatedValues = {...values, team_id: team.team_id}; - onSubmit(updatedValues); - form.resetFields(); - }) - .catch((error) => { - console.error("Validation failed:", error); - }); -}; - - return ( + return ( -
- <> - - - - - - - {"All Proxy Models"} - - {userModels && userModels.map((model) => ( - - {model} - - ))} - - - - - - - - - - - - - - -
- Edit Team -
-
- - ); -}; - -const handleEditClick = (team: any) => { - setSelectedTeam(team); - setEditModalVisible(true); -}; - -const handleEditCancel = () => { - setEditModalVisible(false); - setSelectedTeam(null); -}; - -const handleEditSubmit = async (formValues: Record) => { - // Call API to update team with teamId and values - const teamId = formValues.team_id; // get team_id - - console.log("handleEditSubmit:", formValues); - if (accessToken == null) { - return; - } - - let newTeamValues = await teamUpdateCall(accessToken, formValues); - - // Update the teams state with the updated team data - if (teams) { - const updatedTeams = teams.map((team) => - team.team_id === teamId ? newTeamValues.data : team +
+ <> + + + + + + + {"All Proxy Models"} + + {userModels && + userModels.map((model) => ( + + {model} + + ))} + + + + + + + + + + + + + +
+ Edit Team +
+
+ ); - setTeams(updatedTeams); - } - message.success("Team updated successfully"); + }; - setEditModalVisible(false); - setSelectedTeam(null); -}; + const handleEditClick = (team: any) => { + setSelectedTeam(team); + setEditModalVisible(true); + }; + + const handleEditCancel = () => { + setEditModalVisible(false); + setSelectedTeam(null); + }; + + const handleEditSubmit = async (formValues: Record) => { + // Call API to update team with teamId and values + const teamId = formValues.team_id; // get team_id + + console.log("handleEditSubmit:", formValues); + if (accessToken == null) { + return; + } + + let newTeamValues = await teamUpdateCall(accessToken, formValues); + + // Update the teams state with the updated team data + if (teams) { + const updatedTeams = teams.map((team) => + team.team_id === teamId ? newTeamValues.data : team + ); + setTeams(updatedTeams); + } + message.success("Team updated successfully"); + + setEditModalVisible(false); + setSelectedTeam(null); + }; const handleOk = () => { setIsTeamModalVisible(false); @@ -224,9 +231,6 @@ const handleEditSubmit = async (formValues: Record) => { setIsDeleteModalOpen(true); }; - - - const confirmDelete = async () => { if (teamToDelete == null || teams == null || accessToken == null) { return; @@ -235,7 +239,9 @@ const handleEditSubmit = async (formValues: Record) => { try { await teamDeleteCall(accessToken, teamToDelete); // Successfully completed the deletion. Update the state to trigger a rerender. - const filteredData = teams.filter((item) => item.team_id !== teamToDelete); + const filteredData = teams.filter( + (item) => item.team_id !== teamToDelete + ); setTeams(filteredData); } catch (error) { console.error("Error deleting the team:", error); @@ -253,8 +259,6 @@ const handleEditSubmit = async (formValues: Record) => { setTeamToDelete(null); }; - - useEffect(() => { const fetchUserModels = async () => { try { @@ -263,7 +267,11 @@ const handleEditSubmit = async (formValues: Record) => { } if (accessToken !== null) { - const model_available = await modelAvailableCall(accessToken, userID, userRole); + const model_available = await modelAvailableCall( + accessToken, + userID, + userRole + ); let available_model_names = model_available["data"].map( (element: { id: string }) => element.id ); @@ -275,7 +283,6 @@ const handleEditSubmit = async (formValues: Record) => { } }; - const fetchTeamInfo = async () => { try { if (userID === null || userRole === null || accessToken === null) { @@ -288,22 +295,21 @@ const handleEditSubmit = async (formValues: Record) => { console.log("fetching team info:"); - let _team_id_to_info: Record = {}; for (let i = 0; i < teams?.length; i++) { let _team_id = teams[i].team_id; const teamInfo = await teamInfoCall(accessToken, _team_id); console.log("teamInfo response:", teamInfo); if (teamInfo !== null) { - _team_id_to_info = {..._team_id_to_info, [_team_id]: teamInfo}; + _team_id_to_info = { ..._team_id_to_info, [_team_id]: teamInfo }; } } setPerTeamInfo(_team_id_to_info); - } catch (error) { - console.error("Error fetching team info:", error); - } - }; - + } catch (error) { + console.error("Error fetching team info:", error); + } + }; + fetchUserModels(); fetchTeamInfo(); }, [accessToken, userID, userRole, teams]); @@ -311,6 +317,15 @@ const handleEditSubmit = async (formValues: Record) => { const handleCreate = async (formValues: Record) => { try { if (accessToken != null) { + const newTeamAlias = formValues?.team_alias; + const existingTeamAliases = teams?.map((t) => t.team_alias) ?? []; + + if (existingTeamAliases.includes(newTeamAlias)) { + throw new Error( + `Team alias ${newTeamAlias} already exists, please pick another alias` + ); + } + message.info("Creating Team"); const response: any = await teamCreateCall(accessToken, formValues); if (teams !== null) { @@ -364,7 +379,7 @@ const handleEditSubmit = async (formValues: Record) => { console.error("Error creating the team:", error); } }; - console.log(`received teams ${teams}`); + console.log(`received teams ${JSON.stringify(teams)}`); return (
@@ -387,55 +402,124 @@ const handleEditSubmit = async (formValues: Record) => { {teams && teams.length > 0 ? teams.map((team: any) => ( - {team["team_alias"]} - {team["spend"]} - + + {team["team_alias"]} + + + {team["spend"]} + + {team["max_budget"] ? team["max_budget"] : "No limit"} - + {Array.isArray(team.models) ? ( -
+
{team.models.length === 0 ? ( All Proxy Models ) : ( - team.models.map((model: string, index: number) => ( - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - {model.length > 30 ? `${model.slice(0, 30)}...` : model} - - ) - )) + team.models.map( + (model: string, index: number) => + model === "all-proxy-models" ? ( + + All Proxy Models + + ) : ( + + + {model.length > 30 + ? `${model.slice(0, 30)}...` + : model} + + + ) + ) )}
) : null} - - + - TPM:{" "} - {team.tpm_limit ? team.tpm_limit : "Unlimited"}{" "} + TPM: {team.tpm_limit ? team.tpm_limit : "Unlimited"}{" "}

RPM:{" "} {team.rpm_limit ? team.rpm_limit : "Unlimited"} - {perTeamInfo && team.team_id && perTeamInfo[team.team_id] && perTeamInfo[team.team_id].keys && perTeamInfo[team.team_id].keys.length} Keys - {perTeamInfo && team.team_id && perTeamInfo[team.team_id] && perTeamInfo[team.team_id].team_info && perTeamInfo[team.team_id].team_info.members_with_roles && perTeamInfo[team.team_id].team_info.members_with_roles.length} Members + + {perTeamInfo && + team.team_id && + perTeamInfo[team.team_id] && + perTeamInfo[team.team_id].keys && + perTeamInfo[team.team_id].keys.length}{" "} + Keys + + + {perTeamInfo && + team.team_id && + perTeamInfo[team.team_id] && + perTeamInfo[team.team_id].team_info && + perTeamInfo[team.team_id].team_info + .members_with_roles && + perTeamInfo[team.team_id].team_info + .members_with_roles.length}{" "} + Members + - handleEditClick(team)} /> - handleDelete(team.team_id)} icon={TrashIcon} size="sm" @@ -481,7 +565,11 @@ const handleEditSubmit = async (formValues: Record) => {
- @@ -515,10 +603,12 @@ const handleEditSubmit = async (formValues: Record) => { labelAlign="left" > <> - @@ -528,7 +618,10 @@ const handleEditSubmit = async (formValues: Record) => { placeholder="Select models" style={{ width: "100%" }} > - + All Proxy Models {userModels.map((model) => ( @@ -606,8 +699,8 @@ const handleEditSubmit = async (formValues: Record) => { {member["user_email"] ? member["user_email"] : member["user_id"] - ? member["user_id"] - : null} + ? member["user_id"] + : null} {member["role"]} @@ -618,13 +711,13 @@ const handleEditSubmit = async (formValues: Record) => { {selectedTeam && ( - - )} + + )} - - Key Name: {keyName} - - Monthly Spend ${keySpend} - {predictedSpendString} - - - {data && ( - - )} - - - -
- ); -}; - -export default ViewKeySpendReport; diff --git a/ui/litellm-dashboard/src/components/view_key_table.tsx b/ui/litellm-dashboard/src/components/view_key_table.tsx index d812c19a343..bb6ffc4208c 100644 --- a/ui/litellm-dashboard/src/components/view_key_table.tsx +++ b/ui/litellm-dashboard/src/components/view_key_table.tsx @@ -17,6 +17,7 @@ import { DialogPanel, Text, Title, + Subtitle, Icon, BarChart, } from "@tremor/react"; @@ -32,8 +33,6 @@ import { Select, } from "antd"; -import ViewKeySpendReport from "./view_key_spend_report"; - const { Option } = Select; @@ -570,6 +569,7 @@ const ViewKeyTable: React.FC = ({ icon={InformationCircleIcon} size="sm" /> + = ({ setOpenDialogId(null); setSelectedItem(null); }} + static={true} > @@ -639,22 +640,14 @@ const ViewKeyTable: React.FC = ({
- - {spendData && ( - - )} - - + + Token Name + {selectedItem.key_alias ? selectedItem.key_alias : selectedItem.key_name} + Token ID + {selectedItem.token} Metadata - - {JSON.stringify(selectedItem.metadata)} + 
{JSON.stringify(selectedItem.metadata)}
+ )} - - + + Date: Sat, 11 May 2024 13:08:16 -0700 Subject: [PATCH 106/169] feat - router async batch acompletion --- litellm/router.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index f0d94908e98..7396dab2085 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -605,6 +605,33 @@ class Router: self.fail_calls[model_name] += 1 raise e + async def abatch_completion( + self, models: List[str], messages: List[Dict[str, str]], **kwargs + ): + + async def _async_completion_no_exceptions( + model: str, messages: List[Dict[str, str]], **kwargs + ): + """ + Wrapper around self.async_completion that catches exceptions and returns them as a result + """ + try: + return await self.acompletion(model=model, messages=messages, **kwargs) + except Exception as e: + return e + + _tasks = [] + for model in models: + # add each task but if the task fails + _tasks.append( + _async_completion_no_exceptions( + model=model, messages=messages, **kwargs + ) + ) + + response = await asyncio.gather(*_tasks) + return response + def image_generation(self, prompt: str, model: str, **kwargs): try: kwargs["model"] = model From 6561e0838ea36695847d38f648b81c46d225ce08 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 13:09:17 -0700 Subject: [PATCH 107/169] test - router.batch_acompletion --- litellm/tests/test_router_batch_completion.py | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 litellm/tests/test_router_batch_completion.py diff --git a/litellm/tests/test_router_batch_completion.py b/litellm/tests/test_router_batch_completion.py new file mode 100644 index 00000000000..f2873b18d55 --- /dev/null +++ b/litellm/tests/test_router_batch_completion.py @@ -0,0 +1,60 @@ +#### What this tests #### +# This tests litellm router with batch completion + +import sys, os, time, openai +import traceback, asyncio +import pytest + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import litellm +from litellm import Router +from litellm.router import Deployment, LiteLLM_Params, ModelInfo +from concurrent.futures import ThreadPoolExecutor +from collections import defaultdict +from dotenv import load_dotenv +import os, httpx + +load_dotenv() + + +@pytest.mark.asyncio +async def test_batch_completion_multiple_models(): + litellm.set_verbose = True + + router = litellm.Router( + model_list=[ + { + "model_name": "gpt-3.5-turbo", + "litellm_params": { + "model": "gpt-3.5-turbo", + }, + }, + { + "model_name": "groq-llama", + "litellm_params": { + "model": "groq/llama3-8b-8192", + }, + }, + ] + ) + + response = await router.abatch_completion( + models=["gpt-3.5-turbo", "groq-llama"], + messages=[ + {"role": "user", "content": "is litellm becoming a better product ?"} + ], + max_tokens=15, + ) + + print(response) + assert len(response) == 2 + + models_in_responses = [] + for individual_response in response: + _model = individual_response["model"] + models_in_responses.append(_model) + + # assert both models are different + assert models_in_responses[0] != models_in_responses[1] From b8c7bbcb9f8d10103ebed09f9c73d6d49ca48024 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 13:24:25 -0700 Subject: [PATCH 108/169] support batch /chat/completions on proxy --- litellm/proxy/proxy_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 46c13277328..0ae498baa9f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3639,7 +3639,7 @@ async def chat_completion( ### MODEL ALIAS MAPPING ### # check if model name in model alias map # get the actual model name - if data["model"] in litellm.model_alias_map: + if isinstance(data["model"], str) and data["model"] in litellm.model_alias_map: data["model"] = litellm.model_alias_map[data["model"]] ## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call @@ -3673,6 +3673,9 @@ async def chat_completion( # skip router if user passed their key if "api_key" in data: tasks.append(litellm.acompletion(**data)) + elif isinstance(data["model"], list) and llm_router is not None: + _models = data.pop("model") + tasks.append(llm_router.abatch_completion(models=_models, **data)) elif "user_config" in data: # initialize a new router instance. make request using this Router router_config = data.pop("user_config") From 31cb1be27929eec766447616942b5e7b70f2b61d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 13:24:59 -0700 Subject: [PATCH 109/169] edit dev config.yaml --- litellm/proxy/proxy_config.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index b1cbf2e8186..85634c5b867 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -4,6 +4,12 @@ model_list: model: openai/fake api_key: fake-key api_base: https://exampleopenaiendpoint-production.up.railway.app/ + - model_name: llama3 + litellm_params: + model: groq/llama3-8b-8192 + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo - model_name: "*" litellm_params: model: openai/* From e1f94fcbbb3a51107fa14ef09d69132eff92e124 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 13:32:30 -0700 Subject: [PATCH 110/169] test batch completions on litellm proxy --- tests/test_openai_endpoints.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 38e87c25409..7bc97ca5930 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -4,6 +4,7 @@ import pytest import asyncio import aiohttp, openai from openai import OpenAI, AsyncOpenAI +from typing import Optional, List, Union def response_header_check(response): @@ -71,7 +72,7 @@ async def new_user(session): return await response.json() -async def chat_completion(session, key, model="gpt-4"): +async def chat_completion(session, key, model: Union[str, List] = "gpt-4"): url = "http://0.0.0.0:4000/chat/completions" headers = { "Authorization": f"Bearer {key}", @@ -409,3 +410,27 @@ async def test_openai_wildcard_chat_completion(): # call chat/completions with a model that the key was not created for + the model is not on the config.yaml await chat_completion(session=session, key=key, model="gpt-3.5-turbo-0125") + + +@pytest.mark.asyncio +async def test_batch_chat_completions(): + """ + - Make chat completion call using + + """ + async with aiohttp.ClientSession() as session: + + # call chat/completions with a model that the key was not created for + the model is not on the config.yaml + response = await chat_completion( + session=session, + key="sk-1234", + model=[ + "gpt-3.5-turbo", + "fake-openai-endpoint", + ], + ) + + print(f"response: {response}") + + assert len(response) == 2 + assert isinstance(response, list) From 5918ee543bd9d3523bfc4fdb6c6ceff4384f8a3e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 13:42:41 -0700 Subject: [PATCH 111/169] docs - batch completion litellm proxy --- docs/my-website/docs/proxy/user_keys.md | 84 +++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/my-website/docs/proxy/user_keys.md b/docs/my-website/docs/proxy/user_keys.md index fa78b37c11f..7aba832eb8a 100644 --- a/docs/my-website/docs/proxy/user_keys.md +++ b/docs/my-website/docs/proxy/user_keys.md @@ -365,6 +365,90 @@ curl --location 'http://0.0.0.0:4000/moderations' \ ## Advanced +### (BETA) Batch Completions - pass `model` as List + +Use this when you want to send 1 request to N Models + +#### Expected Request Format + +This same request will be sent to the following model groups on the [litellm proxy config.yaml](https://docs.litellm.ai/docs/proxy/configs) +- `model_name="llama3"` +- `model_name="gpt-3.5-turbo"` + +```shell +curl --location 'http://localhost:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": ["llama3", "gpt-3.5-turbo"], + "max_tokens": 10, + "user": "litellm2", + "messages": [ + { + "role": "user", + "content": "is litellm getting better" + } + ] +}' +``` + + +#### Expected Response Format + +Get a list of responses when `model` is passed as a list + +```json +[ + { + "id": "chatcmpl-3dbd5dd8-7c82-4ca3-bf1f-7c26f497cf2b", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "The Elder Scrolls IV: Oblivion!\n\nReleased", + "role": "assistant" + } + } + ], + "created": 1715459876, + "model": "groq/llama3-8b-8192", + "object": "chat.completion", + "system_fingerprint": "fp_179b0f92c9", + "usage": { + "completion_tokens": 10, + "prompt_tokens": 12, + "total_tokens": 22 + } + }, + { + "id": "chatcmpl-9NnldUfFLmVquFHSX4yAtjCw8PGei", + "choices": [ + { + "finish_reason": "length", + "index": 0, + "message": { + "content": "TES4 could refer to The Elder Scrolls IV:", + "role": "assistant" + } + } + ], + "created": 1715459877, + "model": "gpt-3.5-turbo-0125", + "object": "chat.completion", + "system_fingerprint": null, + "usage": { + "completion_tokens": 10, + "prompt_tokens": 9, + "total_tokens": 19 + } + } +] +``` + + + + ### Pass User LLM API Keys, Fallbacks Allow your end-users to pass their model list, api base, OpenAI API key (any LiteLLM supported provider) to make requests From 4a3b08496129841597b1188820a3c45a01ee9abf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 May 2024 13:43:08 -0700 Subject: [PATCH 112/169] feat(bedrock_httpx.py): moves to using httpx client for bedrock cohere calls --- .../generic_api_callback.py | 3 - litellm/integrations/aispend.py | 2 - litellm/integrations/berrispend.py | 1 - litellm/integrations/clickhouse.py | 4 - litellm/integrations/custom_logger.py | 2 - litellm/integrations/datadog.py | 2 - litellm/integrations/dynamodb.py | 2 - litellm/integrations/helicone.py | 2 - litellm/integrations/langfuse.py | 4 +- litellm/integrations/langsmith.py | 2 - litellm/integrations/lunary.py | 9 +- litellm/integrations/openmeter.py | 2 - litellm/integrations/prometheus.py | 2 - litellm/integrations/prometheus_services.py | 2 - litellm/integrations/prompt_layer.py | 2 - litellm/integrations/s3.py | 4 +- litellm/integrations/slack_alerting.py | 2 - litellm/integrations/supabase.py | 2 - litellm/integrations/weights_biases.py | 11 +- litellm/llms/bedrock_httpx.py | 124 ++++++++++++++++++ litellm/main.py | 3 +- .../proxy/example_config_yaml/custom_auth.py | 3 - litellm/router_strategy/least_busy.py | 2 - litellm/router_strategy/lowest_cost.py | 3 +- litellm/router_strategy/lowest_latency.py | 2 - litellm/router_strategy/lowest_tpm_rpm.py | 2 - litellm/router_strategy/lowest_tpm_rpm_v2.py | 2 - litellm/tests/test_completion.py | 9 ++ litellm/utils.py | 1 - 29 files changed, 147 insertions(+), 64 deletions(-) create mode 100644 litellm/llms/bedrock_httpx.py diff --git a/enterprise/enterprise_callbacks/generic_api_callback.py b/enterprise/enterprise_callbacks/generic_api_callback.py index 076c13d5eef..cf1d22e8f8d 100644 --- a/enterprise/enterprise_callbacks/generic_api_callback.py +++ b/enterprise/enterprise_callbacks/generic_api_callback.py @@ -10,7 +10,6 @@ from litellm.caching import DualCache from typing import Literal, Union -dotenv.load_dotenv() # Loading env variables using dotenv import traceback @@ -19,8 +18,6 @@ import traceback import dotenv, os import requests - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/aispend.py b/litellm/integrations/aispend.py index a893f8923bb..2fe8ea0dfa7 100644 --- a/litellm/integrations/aispend.py +++ b/litellm/integrations/aispend.py @@ -1,8 +1,6 @@ #### What this does #### # On success + failure, log events to aispend.io import dotenv, os - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime diff --git a/litellm/integrations/berrispend.py b/litellm/integrations/berrispend.py index 1f0ae4581fb..7d30b706c8f 100644 --- a/litellm/integrations/berrispend.py +++ b/litellm/integrations/berrispend.py @@ -3,7 +3,6 @@ import dotenv, os import requests # type: ignore -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 7d1fb37d945..0c38b862679 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -8,8 +8,6 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.caching import DualCache from typing import Literal, Union - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback @@ -18,8 +16,6 @@ import traceback import dotenv, os import requests - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 8a3e0f4673c..d508825922e 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -6,8 +6,6 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.caching import DualCache from typing import Literal, Union, Optional - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback diff --git a/litellm/integrations/datadog.py b/litellm/integrations/datadog.py index d969341fc45..6d5e08faffc 100644 --- a/litellm/integrations/datadog.py +++ b/litellm/integrations/datadog.py @@ -3,8 +3,6 @@ import dotenv, os import requests # type: ignore - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/dynamodb.py b/litellm/integrations/dynamodb.py index b5462ee7fa2..21ccabe4b77 100644 --- a/litellm/integrations/dynamodb.py +++ b/litellm/integrations/dynamodb.py @@ -3,8 +3,6 @@ import dotenv, os import requests # type: ignore - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/helicone.py b/litellm/integrations/helicone.py index c8c1075419b..85e73258ea8 100644 --- a/litellm/integrations/helicone.py +++ b/litellm/integrations/helicone.py @@ -3,8 +3,6 @@ import dotenv, os import requests # type: ignore import litellm - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 1e957dfcf88..f27d1996800 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -1,8 +1,6 @@ #### What this does #### # On success, logs events to Langfuse -import dotenv, os - -dotenv.load_dotenv() # Loading env variables using dotenv +import os import copy import traceback from packaging.version import Version diff --git a/litellm/integrations/langsmith.py b/litellm/integrations/langsmith.py index 8a0fb385227..92e4402155d 100644 --- a/litellm/integrations/langsmith.py +++ b/litellm/integrations/langsmith.py @@ -3,8 +3,6 @@ import dotenv, os # type: ignore import requests # type: ignore from datetime import datetime - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import asyncio import types diff --git a/litellm/integrations/lunary.py b/litellm/integrations/lunary.py index 6ddf2ca5992..52316f31549 100644 --- a/litellm/integrations/lunary.py +++ b/litellm/integrations/lunary.py @@ -2,14 +2,11 @@ # On success + failure, log events to lunary.ai from datetime import datetime, timezone import traceback -import dotenv import importlib import sys import packaging -dotenv.load_dotenv() - # convert to {completion: xx, tokens: xx} def parse_usage(usage): @@ -62,14 +59,16 @@ class LunaryLogger: version = importlib.metadata.version("lunary") # if version < 0.1.43 then raise ImportError if packaging.version.Version(version) < packaging.version.Version("0.1.43"): - print( + print( # noqa "Lunary version outdated. Required: >= 0.1.43. Upgrade via 'pip install lunary --upgrade'" ) raise ImportError self.lunary_client = lunary except ImportError: - print("Lunary not installed. Please install it using 'pip install lunary'") + print( # noqa + "Lunary not installed. Please install it using 'pip install lunary'" + ) # noqa raise ImportError def log_event( diff --git a/litellm/integrations/openmeter.py b/litellm/integrations/openmeter.py index a454739d546..2c470d6f49a 100644 --- a/litellm/integrations/openmeter.py +++ b/litellm/integrations/openmeter.py @@ -3,8 +3,6 @@ import dotenv, os, json import litellm - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback from litellm.integrations.custom_logger import CustomLogger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 577946ce18e..6fbc6ca4cee 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -4,8 +4,6 @@ import dotenv, os import requests # type: ignore - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index d276bb85bab..8fce8930de6 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -5,8 +5,6 @@ import dotenv, os import requests # type: ignore - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/prompt_layer.py b/litellm/integrations/prompt_layer.py index ce610e1ef11..531ed75fe03 100644 --- a/litellm/integrations/prompt_layer.py +++ b/litellm/integrations/prompt_layer.py @@ -3,8 +3,6 @@ import dotenv, os import requests # type: ignore from pydantic import BaseModel - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback diff --git a/litellm/integrations/s3.py b/litellm/integrations/s3.py index d31b1584027..d131e44f0e0 100644 --- a/litellm/integrations/s3.py +++ b/litellm/integrations/s3.py @@ -1,9 +1,7 @@ #### What this does #### # On success + failure, log events to Supabase -import dotenv, os - -dotenv.load_dotenv() # Loading env variables using dotenv +import os import traceback import datetime, subprocess, sys import litellm, uuid diff --git a/litellm/integrations/slack_alerting.py b/litellm/integrations/slack_alerting.py index 07c3585f088..d03922bc1f5 100644 --- a/litellm/integrations/slack_alerting.py +++ b/litellm/integrations/slack_alerting.py @@ -2,8 +2,6 @@ # Class for sending Slack Alerts # import dotenv, os from litellm.proxy._types import UserAPIKeyAuth - -dotenv.load_dotenv() # Loading env variables using dotenv from litellm._logging import verbose_logger, verbose_proxy_logger import litellm, threading from typing import List, Literal, Any, Union, Optional, Dict diff --git a/litellm/integrations/supabase.py b/litellm/integrations/supabase.py index 58beba8a3db..4e6bf517f3d 100644 --- a/litellm/integrations/supabase.py +++ b/litellm/integrations/supabase.py @@ -3,8 +3,6 @@ import dotenv, os import requests # type: ignore - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback import datetime, subprocess, sys import litellm diff --git a/litellm/integrations/weights_biases.py b/litellm/integrations/weights_biases.py index 53e6070a5bf..a56233b22f7 100644 --- a/litellm/integrations/weights_biases.py +++ b/litellm/integrations/weights_biases.py @@ -21,11 +21,11 @@ try: # contains a (known) object attribute object: Literal["chat.completion", "edit", "text_completion"] - def __getitem__(self, key: K) -> V: - ... # pragma: no cover + def __getitem__(self, key: K) -> V: ... # noqa - def get(self, key: K, default: Optional[V] = None) -> Optional[V]: - ... # pragma: no cover + def get( # noqa + self, key: K, default: Optional[V] = None + ) -> Optional[V]: ... # pragma: no cover class OpenAIRequestResponseResolver: def __call__( @@ -173,12 +173,11 @@ except: #### What this does #### # On success, logs events to Langfuse -import dotenv, os +import os import requests import requests from datetime import datetime -dotenv.load_dotenv() # Loading env variables using dotenv import traceback diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py new file mode 100644 index 00000000000..c6b0327e6bf --- /dev/null +++ b/litellm/llms/bedrock_httpx.py @@ -0,0 +1,124 @@ +# What is this? +## Initial implementation of calling bedrock via httpx client (allows for async calls). +## V0 - just covers cohere command-r support + +import os, types +import json +from enum import Enum +import requests, copy # type: ignore +import time +from typing import Callable, Optional, List, Literal, Union +from litellm.utils import ( + ModelResponse, + Usage, + map_finish_reason, + CustomStreamWrapper, + Message, + Choices, + get_secret, +) +import litellm +from .prompt_templates.factory import prompt_factory, custom_prompt +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from .base import BaseLLM +import httpx # type: ignore +from .bedrock import BedrockError + + +class BedrockLLM(BaseLLM): + """ + Example call + + ``` + curl --location --request POST 'https://bedrock-runtime.{aws_region_name}.amazonaws.com/model/{bedrock_model_name}/invoke' \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + --user "$AWS_ACCESS_KEY_ID":"$AWS_SECRET_ACCESS_KEY" \ + --aws-sigv4 "aws:amz:us-east-1:bedrock" \ + --data-raw '{ + "prompt": "Hi", + "temperature": 0, + "p": 0.9, + "max_tokens": 4096 + }' + ``` + """ + + def __init__(self) -> None: + super().__init__() + + def get_credentials( + self, + aws_access_key_id: Optional[str] = None, + aws_secret_access_key: Optional[str] = None, + aws_region_name: Optional[str] = None, + aws_session_name: Optional[str] = None, + aws_profile_name: Optional[str] = None, + aws_role_name: Optional[str] = None, + ): + """ + Return a boto3.Credentials object + """ + import boto3 + + ## CHECK IS 'os.environ/' passed in + params_to_check: List[Optional[str]] = [ + aws_access_key_id, + aws_secret_access_key, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + ] + + # Iterate over parameters and update if needed + for i, param in enumerate(params_to_check): + if param and param.startswith("os.environ/"): + _v = get_secret(param) + if _v is not None and isinstance(_v, str): + params_to_check[i] = _v + # Assign updated values back to parameters + ( + aws_access_key_id, + aws_secret_access_key, + aws_region_name, + aws_session_name, + aws_profile_name, + aws_role_name, + ) = params_to_check + + ### CHECK STS ### + if aws_role_name is not None and aws_session_name is not None: + sts_client = boto3.client( + "sts", + aws_access_key_id=aws_access_key_id, # [OPTIONAL] + aws_secret_access_key=aws_secret_access_key, # [OPTIONAL] + ) + + sts_response = sts_client.assume_role( + RoleArn=aws_role_name, RoleSessionName=aws_session_name + ) + + return sts_response["Credentials"] + elif aws_profile_name is not None: ### CHECK SESSION ### + # uses auth values from AWS profile usually stored in ~/.aws/credentials + client = boto3.Session(profile_name=aws_profile_name) + + return client.get_credentials() + else: + session = boto3.Session( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=aws_region_name, + ) + + return session.get_credentials() + + def completion(self, *args, **kwargs) -> Union[ModelResponse, CustomStreamWrapper]: + ## get credentials + ## generate signature + ## make request + return super().completion(*args, **kwargs) + + def embedding(self, *args, **kwargs): + return super().embedding(*args, **kwargs) diff --git a/litellm/main.py b/litellm/main.py index 9afdc7da200..8be71de0b7e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -75,6 +75,7 @@ from .llms.anthropic import AnthropicChatCompletion from .llms.anthropic_text import AnthropicTextCompletion from .llms.huggingface_restapi import Huggingface from .llms.predibase import PredibaseChatCompletion +from .llms.bedrock_httpx import BedrockLLM from .llms.triton import TritonChatCompletion from .llms.prompt_templates.factory import ( prompt_factory, @@ -104,7 +105,6 @@ from litellm.utils import ( ) ####### ENVIRONMENT VARIABLES ################### -dotenv.load_dotenv() # Loading env variables using dotenv openai_chat_completions = OpenAIChatCompletion() openai_text_completions = OpenAITextCompletion() anthropic_chat_completions = AnthropicChatCompletion() @@ -114,6 +114,7 @@ azure_text_completions = AzureTextCompletion() huggingface = Huggingface() predibase_chat_completions = PredibaseChatCompletion() triton_chat_completions = TritonChatCompletion() +bedrock_chat_completion = BedrockLLM() ####### COMPLETION ENDPOINTS ################ diff --git a/litellm/proxy/example_config_yaml/custom_auth.py b/litellm/proxy/example_config_yaml/custom_auth.py index a764a647a96..6cecf466c10 100644 --- a/litellm/proxy/example_config_yaml/custom_auth.py +++ b/litellm/proxy/example_config_yaml/custom_auth.py @@ -1,10 +1,7 @@ from litellm.proxy._types import UserAPIKeyAuth, GenerateKeyRequest from fastapi import Request -from dotenv import load_dotenv import os -load_dotenv() - async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: try: diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py index 54d44b41d54..417651fb3e7 100644 --- a/litellm/router_strategy/least_busy.py +++ b/litellm/router_strategy/least_busy.py @@ -8,8 +8,6 @@ import dotenv, os, requests, random # type: ignore from typing import Optional - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index 279af2ae9c9..fde7781b9b0 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -1,12 +1,11 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) from pydantic import BaseModel, Extra, Field, root_validator -import dotenv, os, requests, random # type: ignore +import os, requests, random # type: ignore from typing import Optional, Union, List, Dict from datetime import datetime, timedelta import random -dotenv.load_dotenv() # Loading env variables using dotenv import traceback from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index afdfc177934..a7b93d344db 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -5,8 +5,6 @@ import dotenv, os, requests, random # type: ignore from typing import Optional, Union, List, Dict from datetime import datetime, timedelta import random - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback from litellm.caching import DualCache from litellm.integrations.custom_logger import CustomLogger diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 0a7773a84b8..625db70482d 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -4,8 +4,6 @@ import dotenv, os, requests, random from typing import Optional, Union, List, Dict from datetime import datetime - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback from litellm import token_counter from litellm.caching import DualCache diff --git a/litellm/router_strategy/lowest_tpm_rpm_v2.py b/litellm/router_strategy/lowest_tpm_rpm_v2.py index f7a55d97091..23e55f4a3c4 100644 --- a/litellm/router_strategy/lowest_tpm_rpm_v2.py +++ b/litellm/router_strategy/lowest_tpm_rpm_v2.py @@ -5,8 +5,6 @@ import dotenv, os, requests, random from typing import Optional, Union, List, Dict import datetime as datetime_og from datetime import datetime - -dotenv.load_dotenv() # Loading env variables using dotenv import traceback, asyncio, httpx import litellm from litellm import token_counter diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 04f4cc5115c..214dc105b11 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2584,6 +2584,15 @@ def test_completion_chat_sagemaker_mistral(): # test_completion_chat_sagemaker_mistral() +def test_completion_bedrock_command_r(): + response = completion( + model="bedrock/cohere.command-r-plus-v1:0", + messages=[{"role": "user", "content": "Hey! how's it going?"}], + ) + + print(f"response: {response}") + + def test_completion_bedrock_titan_null_response(): try: response = completion( diff --git a/litellm/utils.py b/litellm/utils.py index 9218f92a3e7..0fd7963ae32 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -117,7 +117,6 @@ MAX_THREADS = 100 # Create a ThreadPoolExecutor executor = ThreadPoolExecutor(max_workers=MAX_THREADS) -dotenv.load_dotenv() # Loading env variables using dotenv sentry_sdk_instance = None capture_exception = None add_breadcrumb = None From 62276fc22102aafbc9ea91d0f978a39b252ce75a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 13:45:32 -0700 Subject: [PATCH 113/169] docs link to litellm batch completions --- docs/my-website/docs/completion/batching.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/my-website/docs/completion/batching.md b/docs/my-website/docs/completion/batching.md index 05683b3ddbf..09f59f743d0 100644 --- a/docs/my-website/docs/completion/batching.md +++ b/docs/my-website/docs/completion/batching.md @@ -4,6 +4,12 @@ LiteLLM allows you to: * Send 1 completion call to many models: Return Fastest Response * Send 1 completion call to many models: Return All Responses +:::info + +Trying to do batch completion on LiteLLM Proxy ? Go here: https://docs.litellm.ai/docs/proxy/user_keys#beta-batch-completions---pass-model-as-list + +::: + ## Send multiple completion calls to 1 model In the batch_completion method, you provide a list of `messages` where each sub-list of messages is passed to `litellm.completion()`, allowing you to process multiple prompts efficiently in a single API call. From 038522ab24d92e0644834dcc2ee0b9d7d5899417 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:02:16 -0700 Subject: [PATCH 114/169] fix - support debugging litellm params --- litellm/integrations/langfuse.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index bc1d501906c..b9c8fafc9a8 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -318,11 +318,12 @@ class LangFuseLogger: else: clean_metadata[key] = value - session_id = clean_metadata.get("session_id", None) - trace_name = clean_metadata.get("trace_name", None) - trace_id = clean_metadata.get("trace_id", None) - existing_trace_id = clean_metadata.get("existing_trace_id", None) - update_trace_keys = clean_metadata.get("update_trace_keys", []) + session_id = clean_metadata.pop("session_id", None) + trace_name = clean_metadata.pop("trace_name", None) + trace_id = clean_metadata.pop("trace_id", None) + existing_trace_id = clean_metadata.pop("existing_trace_id", None) + update_trace_keys = clean_metadata.pop("update_trace_keys", []) + debug = clean_metadata.pop("debug", None) if trace_name is None and existing_trace_id is None: # just log `litellm-{call_type}` as the trace name @@ -342,10 +343,6 @@ class LangFuseLogger: if updated_trace_value is not None: trace_params[trace_param_key] = updated_trace_value - if "metadata" in update_trace_keys: - # log metadata in the trace - trace_params["metadata"] = clean_metadata - # Pop the trace specific keys that would have been popped if there were a new trace for key in list( filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) @@ -367,7 +364,6 @@ class LangFuseLogger: "trace_version", clean_metadata.get("version", None) ), # If provided just version, it will applied to the trace as well, if applied a trace version it will take precedence "user_id": user_id, - "metadata": clean_metadata, } for key in list( filter(lambda key: key.startswith("trace_"), clean_metadata.keys()) @@ -381,6 +377,13 @@ class LangFuseLogger: else: trace_params["output"] = output + if debug == True: + if "metadata" in trace_params: + # log the raw_metadata in the trace + trace_params["metadata"]["metadata_passed_to_litellm"] = metadata + else: + trace_params["metadata"] = {"metadata_passed_to_litellm": metadata} + cost = kwargs.get("response_cost", None) print_verbose(f"trace: {cost}") From 97c81a5c7ed8b25a22c09d1f87177cf40394ae27 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:03:40 -0700 Subject: [PATCH 115/169] fix langfuse test --- litellm/tests/test_alangfuse.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/tests/test_alangfuse.py b/litellm/tests/test_alangfuse.py index 6f9b8d6265d..4eeba072148 100644 --- a/litellm/tests/test_alangfuse.py +++ b/litellm/tests/test_alangfuse.py @@ -247,7 +247,9 @@ async def test_langfuse_logging_metadata(langfuse_client): expected_filtered_metadata_keys = { "trace_name", "trace_id", + "existing_trace_id", "trace_user_id", + "session_id", "tags", "generation_name", "generation_id", From 1bf8e7ac75f0d44e911801f7ce23c9944f1d2f79 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:08:39 -0700 Subject: [PATCH 116/169] fix langfuse debug mode --- litellm/integrations/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index b9c8fafc9a8..90c274a90e9 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -377,7 +377,7 @@ class LangFuseLogger: else: trace_params["output"] = output - if debug == True: + if debug == True or (isinstance(debug, str) and debug.lower() == "true"): if "metadata" in trace_params: # log the raw_metadata in the trace trace_params["metadata"]["metadata_passed_to_litellm"] = metadata From 360d2840583371b27b03f038d401a2d86db8276f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:12:17 -0700 Subject: [PATCH 117/169] docs - debug langfuse --- docs/my-website/docs/observability/langfuse_integration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/observability/langfuse_integration.md b/docs/my-website/docs/observability/langfuse_integration.md index ebf20b63359..7ba204497a9 100644 --- a/docs/my-website/docs/observability/langfuse_integration.md +++ b/docs/my-website/docs/observability/langfuse_integration.md @@ -136,6 +136,7 @@ response = completion( "existing_trace_id": "trace-id22", "trace_metadata": {"key": "updated_trace_value"}, # The new value to use for the langfuse Trace Metadata "update_trace_keys": ["input", "output", "trace_metadata"], # Updates the trace input & output to be this generations input & output also updates the Trace Metadata to match the passed in value + "debug_langfuse": True, # Will log the exact metadata sent to litellm for the trace/generation as `metadata_passed_to_litellm` }, ) From a41bef52971528b1b740f9ccd5e7c3dacb132ea2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 14:12:26 -0700 Subject: [PATCH 118/169] debug langfuse --- litellm/integrations/langfuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index 90c274a90e9..5cdf83a7c2f 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -323,7 +323,7 @@ class LangFuseLogger: trace_id = clean_metadata.pop("trace_id", None) existing_trace_id = clean_metadata.pop("existing_trace_id", None) update_trace_keys = clean_metadata.pop("update_trace_keys", []) - debug = clean_metadata.pop("debug", None) + debug = clean_metadata.pop("debug_langfuse", None) if trace_name is None and existing_trace_id is None: # just log `litellm-{call_type}` as the trace name From 59c8c0adff167d373643e3872f2b4e5f5142fa81 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Sat, 11 May 2024 15:04:38 -0700 Subject: [PATCH 119/169] feat(bedrock_httpx.py): working cohere command r async calls --- litellm/__init__.py | 1 + litellm/llms/bedrock_httpx.py | 316 +++++++++++++++++++++- litellm/llms/custom_httpx/http_handler.py | 27 +- litellm/main.py | 45 ++- litellm/tests/test_completion.py | 1 + litellm/types/llms/bedrock.py | 6 + 6 files changed, 364 insertions(+), 32 deletions(-) create mode 100644 litellm/types/llms/bedrock.py diff --git a/litellm/__init__.py b/litellm/__init__.py index aedf4213917..67170c68d0a 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -670,6 +670,7 @@ from .llms.sagemaker import SagemakerConfig from .llms.ollama import OllamaConfig from .llms.ollama_chat import OllamaChatConfig from .llms.maritalk import MaritTalkConfig +from .llms.bedrock_httpx import AmazonCohereChatConfig from .llms.bedrock import ( AmazonTitanConfig, AmazonAI21Config, diff --git a/litellm/llms/bedrock_httpx.py b/litellm/llms/bedrock_httpx.py index c6b0327e6bf..d3062b5ed84 100644 --- a/litellm/llms/bedrock_httpx.py +++ b/litellm/llms/bedrock_httpx.py @@ -7,7 +7,7 @@ import json from enum import Enum import requests, copy # type: ignore import time -from typing import Callable, Optional, List, Literal, Union +from typing import Callable, Optional, List, Literal, Union, Any, TypedDict, Tuple from litellm.utils import ( ModelResponse, Usage, @@ -18,11 +18,110 @@ from litellm.utils import ( get_secret, ) import litellm -from .prompt_templates.factory import prompt_factory, custom_prompt -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from .prompt_templates.factory import prompt_factory, custom_prompt, cohere_message_pt +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from .base import BaseLLM import httpx # type: ignore -from .bedrock import BedrockError +from .bedrock import BedrockError, convert_messages_to_prompt +from litellm.types.llms.bedrock import * + + +class AmazonCohereChatConfig: + """ + Reference - https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-cohere-command-r-plus.html + """ + + documents: Optional[List[Document]] = None + search_queries_only: Optional[bool] = None + preamble: Optional[str] = None + max_tokens: Optional[int] = None + temperature: Optional[float] = None + p: Optional[float] = None + k: Optional[float] = None + prompt_truncation: Optional[str] = None + frequency_penalty: Optional[float] = None + presence_penalty: Optional[float] = None + seed: Optional[int] = None + return_prompt: Optional[bool] = None + stop_sequences: Optional[List[str]] = None + raw_prompting: Optional[bool] = None + + def __init__( + self, + documents: Optional[List[Document]] = None, + search_queries_only: Optional[bool] = None, + preamble: Optional[str] = None, + max_tokens: Optional[int] = None, + temperature: Optional[float] = None, + p: Optional[float] = None, + k: Optional[float] = None, + prompt_truncation: Optional[str] = None, + frequency_penalty: Optional[float] = None, + presence_penalty: Optional[float] = None, + seed: Optional[int] = None, + return_prompt: Optional[bool] = None, + stop_sequences: Optional[str] = None, + raw_prompting: Optional[bool] = None, + ) -> None: + locals_ = locals() + for key, value in locals_.items(): + if key != "self" and value is not None: + setattr(self.__class__, key, value) + + @classmethod + def get_config(cls): + return { + k: v + for k, v in cls.__dict__.items() + if not k.startswith("__") + and not isinstance( + v, + ( + types.FunctionType, + types.BuiltinFunctionType, + classmethod, + staticmethod, + ), + ) + and v is not None + } + + def get_supported_openai_params(self) -> List[str]: + return [ + "max_tokens", + "stream", + "stop", + "temperature", + "top_p", + "frequency_penalty", + "presence_penalty", + "seed", + "stop", + ] + + def map_openai_params( + self, non_default_params: dict, optional_params: dict + ) -> dict: + for param, value in non_default_params.items(): + if param == "max_tokens": + optional_params["max_tokens"] = value + if param == "stream": + optional_params["stream"] = value + if param == "stop": + if isinstance(value, str): + value = [value] + optional_params["stop_sequences"] = value + if param == "temperature": + optional_params["temperature"] = value + if param == "top_p": + optional_params["p"] = value + if param == "frequency_penalty": + optional_params["frequency_penalty"] = value + if param == "presence_penalty": + optional_params["presence_penalty"] = value + if "seed": + optional_params["seed"] = value + return optional_params class BedrockLLM(BaseLLM): @@ -47,6 +146,48 @@ class BedrockLLM(BaseLLM): def __init__(self) -> None: super().__init__() + def convert_messages_to_prompt( + self, model, messages, provider, custom_prompt_dict + ) -> Tuple[str, Optional[list]]: + # handle anthropic prompts and amazon titan prompts + prompt = "" + chat_history: Optional[list] = None + if provider == "anthropic" or provider == "amazon": + if model in custom_prompt_dict: + # check if the model has a registered custom prompt + model_prompt_details = custom_prompt_dict[model] + prompt = custom_prompt( + role_dict=model_prompt_details["roles"], + initial_prompt_value=model_prompt_details["initial_prompt_value"], + final_prompt_value=model_prompt_details["final_prompt_value"], + messages=messages, + ) + else: + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "mistral": + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "meta": + prompt = prompt_factory( + model=model, messages=messages, custom_llm_provider="bedrock" + ) + elif provider == "cohere": + prompt, chat_history = cohere_message_pt(messages=messages) + else: + prompt = "" + for message in messages: + if "role" in message: + if message["role"] == "user": + prompt += f"{message['content']}" + else: + prompt += f"{message['content']}" + else: + prompt += f"{message['content']}" + return prompt, chat_history # type: ignore + def get_credentials( self, aws_access_key_id: Optional[str] = None, @@ -114,11 +255,168 @@ class BedrockLLM(BaseLLM): return session.get_credentials() - def completion(self, *args, **kwargs) -> Union[ModelResponse, CustomStreamWrapper]: - ## get credentials - ## generate signature - ## make request - return super().completion(*args, **kwargs) + def completion( + self, + model: str, + messages: list, + custom_prompt_dict: dict, + model_response: ModelResponse, + print_verbose: Callable, + encoding, + logging_obj, + optional_params: dict, + timeout: Optional[Union[float, httpx.Timeout]], + litellm_params=None, + logger_fn=None, + extra_headers: Optional[dict] = None, + client: Optional[HTTPHandler] = None, + ) -> Union[ModelResponse, CustomStreamWrapper]: + try: + import boto3 + + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + except ImportError as e: + raise ImportError("Missing boto3 to call bedrock. Run 'pip install boto3'.") + + ## CREDENTIALS ## + # pop aws_secret_access_key, aws_access_key_id, aws_region_name from kwargs, since completion calls fail with them + aws_secret_access_key = optional_params.pop("aws_secret_access_key", None) + aws_access_key_id = optional_params.pop("aws_access_key_id", None) + aws_region_name = optional_params.pop("aws_region_name", None) + aws_role_name = optional_params.pop("aws_role_name", None) + aws_session_name = optional_params.pop("aws_session_name", None) + aws_profile_name = optional_params.pop("aws_profile_name", None) + aws_bedrock_runtime_endpoint = optional_params.pop( + "aws_bedrock_runtime_endpoint", None + ) # https://bedrock-runtime.{region_name}.amazonaws.com + + ### SET REGION NAME ### + if aws_region_name is None: + # check env # + litellm_aws_region_name = get_secret("AWS_REGION_NAME", None) + + if litellm_aws_region_name is not None and isinstance( + litellm_aws_region_name, str + ): + aws_region_name = litellm_aws_region_name + + standard_aws_region_name = get_secret("AWS_REGION", None) + if standard_aws_region_name is not None and isinstance( + standard_aws_region_name, str + ): + aws_region_name = standard_aws_region_name + + if aws_region_name is None: + aws_region_name = "us-west-2" + + credentials: Credentials = self.get_credentials( + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_region_name=aws_region_name, + aws_session_name=aws_session_name, + aws_profile_name=aws_profile_name, + aws_role_name=aws_role_name, + ) + + ### SET RUNTIME ENDPOINT ### + endpoint_url = "" + env_aws_bedrock_runtime_endpoint = get_secret("AWS_BEDROCK_RUNTIME_ENDPOINT") + if aws_bedrock_runtime_endpoint is not None and isinstance( + aws_bedrock_runtime_endpoint, str + ): + endpoint_url = aws_bedrock_runtime_endpoint + elif env_aws_bedrock_runtime_endpoint and isinstance( + env_aws_bedrock_runtime_endpoint, str + ): + endpoint_url = env_aws_bedrock_runtime_endpoint + else: + endpoint_url = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com" + + endpoint_url = f"{endpoint_url}/model/{model}/invoke" + + sigv4 = SigV4Auth(credentials, "bedrock", aws_region_name) + + provider = model.split(".")[0] + prompt, chat_history = self.convert_messages_to_prompt( + model, messages, provider, custom_prompt_dict + ) + inference_params = copy.deepcopy(optional_params) + stream = inference_params.pop("stream", False) + + if provider == "cohere": + if model.startswith("cohere.command-r"): + ## LOAD CONFIG + config = litellm.AmazonCohereChatConfig().get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + if optional_params.get("stream", False) == True: + inference_params["stream"] = ( + True # cohere requires stream = True in inference params + ) + + _data = {"message": prompt, **inference_params} + if chat_history is not None: + _data["chat_history"] = chat_history + data = json.dumps(_data) + else: + ## LOAD CONFIG + config = litellm.AmazonCohereConfig.get_config() + for k, v in config.items(): + if ( + k not in inference_params + ): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in + inference_params[k] = v + if optional_params.get("stream", False) == True: + inference_params["stream"] = ( + True # cohere requires stream = True in inference params + ) + data = json.dumps({"prompt": prompt, **inference_params}) + else: + raise Exception("UNSUPPORTED PROVIDER") + + ## COMPLETION CALL + headers = {"Content-Type": "application/json"} + request = AWSRequest( + method="POST", url=endpoint_url, data=data, headers=headers + ) + sigv4.add_auth(request) + prepped = request.prepare() + + if client is None: + _params = {} + if timeout is not None: + if isinstance(timeout, float) or isinstance(timeout, int): + timeout = httpx.Timeout(timeout) + _params["timeout"] = timeout + self.client = HTTPHandler(**_params) # type: ignore + else: + self.client = client + + ## LOGGING + logging_obj.pre_call( + input=messages, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": prepped.url, + "headers": prepped.headers, + }, + ) + + response = self.client.post(url=prepped.url, headers=prepped.headers, data=data) # type: ignore + + try: + response.raise_for_status() + except httpx.HTTPStatusError as err: + error_code = err.response.status_code + raise BedrockError(status_code=error_code, message=response.text) + + return response def embedding(self, *args, **kwargs): return super().embedding(*args, **kwargs) diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 7c7d4938a40..529ba3b390a 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -58,16 +58,25 @@ class AsyncHTTPHandler: class HTTPHandler: def __init__( - self, timeout: httpx.Timeout = _DEFAULT_TIMEOUT, concurrent_limit=1000 + self, + timeout: Optional[httpx.Timeout] = None, + concurrent_limit=1000, + client: Optional[httpx.Client] = None, ): - # Create a client with a connection pool - self.client = httpx.Client( - timeout=timeout, - limits=httpx.Limits( - max_connections=concurrent_limit, - max_keepalive_connections=concurrent_limit, - ), - ) + if timeout is None: + timeout = _DEFAULT_TIMEOUT + + if client is None: + # Create a client with a connection pool + self.client = httpx.Client( + timeout=timeout, + limits=httpx.Limits( + max_connections=concurrent_limit, + max_keepalive_connections=concurrent_limit, + ), + ) + else: + self.client = client def close(self): # Close the client when you're done with it diff --git a/litellm/main.py b/litellm/main.py index 8be71de0b7e..d2f3939fdee 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1922,20 +1922,37 @@ def completion( elif custom_llm_provider == "bedrock": # boto3 reads keys from .env custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - response = bedrock.completion( - model=model, - messages=messages, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=encoding, - logging_obj=logging, - extra_headers=extra_headers, - timeout=timeout, - ) + + if "cohere" in model: + response = bedrock_chat_completion.completion( + model=model, + messages=messages, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + ) + else: + response = bedrock.completion( + model=model, + messages=messages, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=encoding, + logging_obj=logging, + extra_headers=extra_headers, + timeout=timeout, + ) if ( "stream" in optional_params diff --git a/litellm/tests/test_completion.py b/litellm/tests/test_completion.py index 214dc105b11..0cf6dda8359 100644 --- a/litellm/tests/test_completion.py +++ b/litellm/tests/test_completion.py @@ -2585,6 +2585,7 @@ def test_completion_chat_sagemaker_mistral(): def test_completion_bedrock_command_r(): + litellm.set_verbose = True response = completion( model="bedrock/cohere.command-r-plus-v1:0", messages=[{"role": "user", "content": "Hey! how's it going?"}], diff --git a/litellm/types/llms/bedrock.py b/litellm/types/llms/bedrock.py new file mode 100644 index 00000000000..87ef6fd3cc2 --- /dev/null +++ b/litellm/types/llms/bedrock.py @@ -0,0 +1,6 @@ +from typing import TypedDict + + +class Document(TypedDict): + title: str + snippet: str From 9167ff0d759acbfca0ff98de8d472615faa3c9df Mon Sep 17 00:00:00 2001 From: Marc Abramowitz Date: Sat, 11 May 2024 15:22:30 -0700 Subject: [PATCH 120/169] Set fake env vars for `client_no_auth` fixture This allows all of the tests in `test_proxy_server.py` to pass, with the exception of `test_load_router_config`, without needing to set up real environment variables. Before: ```shell $ env -i PATH=$PATH poetry run pytest litellm/tests/test_proxy_server.py -k 'not test_load_router_config' --disable-warnings ... ========================================================== short test summary info =========================================================== ERROR litellm/tests/test_proxy_server.py::test_bedrock_embedding - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_chat_completion - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_chat_completion_azure - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_chat_completion_optional_params - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_embedding - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_engines_model_chat_completions - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_health - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_img_gen - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ERROR litellm/tests/test_proxy_server.py::test_openai_deployments_model_chat_completions_azure - openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY enviro... ========================================== 2 skipped, 1 deselected, 39 warnings, 9 errors in 3.24s =========================================== ``` After: ```shell $ env -i PATH=$PATH poetry run pytest litellm/tests/test_proxy_server.py -k 'not test_load_router_config' --disable-warnings ============================================================ test session starts ============================================================= platform darwin -- Python 3.12.3, pytest-7.4.4, pluggy-1.5.0 rootdir: /Users/abramowi/Code/OpenSource/litellm plugins: anyio-4.3.0, asyncio-0.23.6, mock-3.14.0 asyncio: mode=Mode.STRICT collected 12 items / 1 deselected / 11 selected litellm/tests/test_proxy_server.py s.........s [100%] ========================================== 9 passed, 2 skipped, 1 deselected, 48 warnings in 8.42s =========================================== ``` --- litellm/tests/test_proxy_server.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/litellm/tests/test_proxy_server.py b/litellm/tests/test_proxy_server.py index c1965dc2aac..76b2b2d185c 100644 --- a/litellm/tests/test_proxy_server.py +++ b/litellm/tests/test_proxy_server.py @@ -109,7 +109,17 @@ def mock_patch_aimage_generation(): @pytest.fixture(scope="function") -def client_no_auth(): +def fake_env_vars(monkeypatch): + # Set some fake environment variables + monkeypatch.setenv("OPENAI_API_KEY", "fake_openai_api_key") + monkeypatch.setenv("OPENAI_API_BASE", "http://fake-openai-api-base") + monkeypatch.setenv("AZURE_API_BASE", "http://fake-azure-api-base") + monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake_azure_openai_api_key") + monkeypatch.setenv("AZURE_SWEDEN_API_BASE", "http://fake-azure-sweden-api-base") + + +@pytest.fixture(scope="function") +def client_no_auth(fake_env_vars): # Assuming litellm.proxy.proxy_server is an object from litellm.proxy.proxy_server import cleanup_router_config_variables From 91a6a0eef40c4f1a9f91a8e78a989f4091fc10c7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 15:57:06 -0700 Subject: [PATCH 121/169] (Fix) - linting errors --- litellm/exceptions.py | 6 +- litellm/proxy/hooks/azure_content_safety.py | 3 +- litellm/utils.py | 71 +++++++++++++++------ 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 7c3471acf36..d239f1e1280 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -202,13 +202,11 @@ class BudgetExceededError(Exception): ## DEPRECATED ## class InvalidRequestError(openai.BadRequestError): # type: ignore - def __init__( - self, message, model, llm_provider, response: Optional[httpx.Response] = None - ): + def __init__(self, message, model, llm_provider): self.status_code = 400 self.message = message self.model = model self.llm_provider = llm_provider super().__init__( - self.message, response=response, body=None + self.message, f"{self.model}" ) # Call the base class constructor with the parameters it needs diff --git a/litellm/proxy/hooks/azure_content_safety.py b/litellm/proxy/hooks/azure_content_safety.py index 433571c15ce..5b5139f8c78 100644 --- a/litellm/proxy/hooks/azure_content_safety.py +++ b/litellm/proxy/hooks/azure_content_safety.py @@ -4,6 +4,7 @@ from litellm.proxy._types import UserAPIKeyAuth import litellm, traceback, sys, uuid from fastapi import HTTPException from litellm._logging import verbose_proxy_logger +from typing import Optional class _PROXY_AzureContentSafety( @@ -71,7 +72,7 @@ class _PROXY_AzureContentSafety( return result - async def test_violation(self, content: str, source: str = None): + async def test_violation(self, content: str, source: Optional[str] = None): verbose_proxy_logger.debug("Testing Azure Content-Safety for: %s", content) # Construct a request diff --git a/litellm/utils.py b/litellm/utils.py index e787d215570..2eee41d8d47 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -35,6 +35,7 @@ from dataclasses import ( import litellm._service_logger # for storing API inputs, outputs, and metadata from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.caching import DualCache + oidc_cache = DualCache() try: @@ -2957,7 +2958,7 @@ def client(original_function): ) else: return result - + return result # Prints Exactly what was passed to litellm function - don't execute any logic here - it should just print @@ -9559,16 +9560,20 @@ def get_secret( if oidc_token is not None: return oidc_token - client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) + oidc_client = HTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) # https://cloud.google.com/compute/docs/instances/verifying-instance-identity#request_signature - response = client.get( + response = oidc_client.get( "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/identity", params={"audience": oidc_aud}, headers={"Metadata-Flavor": "Google"}, ) if response.status_code == 200: oidc_token = response.text - oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=3600 - 60) + oidc_cache.set_cache( + key=secret_name, value=oidc_token, ttl=3600 - 60 + ) return oidc_token else: raise ValueError("Google OIDC provider failed") @@ -9587,25 +9592,34 @@ def get_secret( case "github": # https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-cloud-providers#using-custom-actions actions_id_token_request_url = os.getenv("ACTIONS_ID_TOKEN_REQUEST_URL") - actions_id_token_request_token = os.getenv("ACTIONS_ID_TOKEN_REQUEST_TOKEN") - if actions_id_token_request_url is None or actions_id_token_request_token is None: - raise ValueError("ACTIONS_ID_TOKEN_REQUEST_URL or ACTIONS_ID_TOKEN_REQUEST_TOKEN not found in environment") + actions_id_token_request_token = os.getenv( + "ACTIONS_ID_TOKEN_REQUEST_TOKEN" + ) + if ( + actions_id_token_request_url is None + or actions_id_token_request_token is None + ): + raise ValueError( + "ACTIONS_ID_TOKEN_REQUEST_URL or ACTIONS_ID_TOKEN_REQUEST_TOKEN not found in environment" + ) oidc_token = oidc_cache.get_cache(key=secret_name) if oidc_token is not None: return oidc_token - client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0)) - response = client.get( + oidc_client = HTTPHandler( + timeout=httpx.Timeout(timeout=600.0, connect=5.0) + ) + response = oidc_client.get( actions_id_token_request_url, params={"audience": oidc_aud}, headers={ "Authorization": f"Bearer {actions_id_token_request_token}", "Accept": "application/json; api-version=2.0", - }, + }, ) if response.status_code == 200: - oidc_token = response.text['value'] + oidc_token = response.text["value"] oidc_cache.set_cache(key=secret_name, value=oidc_token, ttl=300 - 5) return oidc_token else: @@ -9613,7 +9627,6 @@ def get_secret( case _: raise ValueError("Unsupported OIDC provider") - try: if litellm.secret_manager_client is not None: try: @@ -10562,7 +10575,12 @@ class CustomStreamWrapper: response = chunk.replace("data: ", "").strip() parsed_response = json.loads(response) else: - return {"text": "", "is_finished": False, "prompt_tokens": 0, "completion_tokens": 0} + return { + "text": "", + "is_finished": False, + "prompt_tokens": 0, + "completion_tokens": 0, + } else: print_verbose(f"chunk: {chunk} (Type: {type(chunk)})") raise ValueError( @@ -10583,19 +10601,32 @@ class CustomStreamWrapper: return {"text": "", "is_finished": False} except Exception as e: raise e - + def handle_clarifai_completion_chunk(self, chunk): try: if isinstance(chunk, dict): - parsed_response = chunk + parsed_response = chunk if isinstance(chunk, (str, bytes)): if isinstance(chunk, bytes): parsed_response = chunk.decode("utf-8") else: parsed_response = chunk - data_json = json.loads(parsed_response) - text = data_json.get("outputs", "")[0].get("data", "").get("text", "").get("raw","") - prompt_tokens = len(encoding.encode(data_json.get("outputs", "")[0].get("input","").get("data", "").get("text", "").get("raw",""))) + data_json = json.loads(parsed_response) + text = ( + data_json.get("outputs", "")[0] + .get("data", "") + .get("text", "") + .get("raw", "") + ) + prompt_tokens = len( + encoding.encode( + data_json.get("outputs", "")[0] + .get("input", "") + .get("data", "") + .get("text", "") + .get("raw", "") + ) + ) completion_tokens = len(encoding.encode(text)) return { "text": text, @@ -10650,9 +10681,7 @@ class CustomStreamWrapper: completion_obj["content"] = response_obj["text"] if response_obj["is_finished"]: self.received_finish_reason = response_obj["finish_reason"] - elif ( - self.custom_llm_provider and self.custom_llm_provider == "clarifai" - ): + elif self.custom_llm_provider and self.custom_llm_provider == "clarifai": response_obj = self.handle_clarifai_completion_chunk(chunk) completion_obj["content"] = response_obj["text"] elif self.model == "replicate" or self.custom_llm_provider == "replicate": From d7f71208800397f3c5d14ba81e5f2c304cc08ae9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Sat, 11 May 2024 15:58:53 -0700 Subject: [PATCH 122/169] ui - new build --- litellm/proxy/_experimental/out/404.html | 2 +- .../_buildManifest.js | 0 .../_ssgManifest.js | 0 ...bc85-17d29013b8ff3da5.js => 2f6dbc85-052c4579f80d66ae.js} | 0 .../{69-e49705773ae41779.js => 69-04708d7d4a17c1ee.js} | 0 .../{566-ccd699ab19124658.js => 884-7576ee407a2ecbe6.js} | 4 ++-- ...nd-a166f65a6af292d9.js => _not-found-b1ee1381b72386c2.js} | 0 ...layout-94ae1345f5d85446.js => layout-a756276e228ca058.js} | 0 .../out/_next/static/chunks/app/page-c804e862b63be987.js | 1 - .../out/_next/static/chunks/app/page-e6190351ac8da62a.js | 1 + ...1056-dafd44dfa2da140c.js => fd9d1056-f960ab1e6d32b002.js} | 0 .../out/_next/static/chunks/webpack-de9c0fadf6a94b3b.js | 2 +- .../_experimental/out/_next/static/css/a1602eb39f799143.css | 5 ----- .../_experimental/out/_next/static/css/f04e46b02318b660.css | 5 +++++ litellm/proxy/_experimental/out/index.html | 2 +- litellm/proxy/_experimental/out/index.txt | 4 ++-- ui/litellm-dashboard/out/404.html | 2 +- .../_buildManifest.js | 0 .../_ssgManifest.js | 0 ...bc85-17d29013b8ff3da5.js => 2f6dbc85-052c4579f80d66ae.js} | 0 .../{69-e49705773ae41779.js => 69-04708d7d4a17c1ee.js} | 0 .../{566-ccd699ab19124658.js => 884-7576ee407a2ecbe6.js} | 4 ++-- ...nd-a166f65a6af292d9.js => _not-found-b1ee1381b72386c2.js} | 0 ...layout-94ae1345f5d85446.js => layout-a756276e228ca058.js} | 0 .../out/_next/static/chunks/app/page-c804e862b63be987.js | 1 - .../out/_next/static/chunks/app/page-e6190351ac8da62a.js | 1 + ...1056-dafd44dfa2da140c.js => fd9d1056-f960ab1e6d32b002.js} | 0 .../out/_next/static/chunks/webpack-de9c0fadf6a94b3b.js | 2 +- .../out/_next/static/css/a1602eb39f799143.css | 5 ----- .../out/_next/static/css/f04e46b02318b660.css | 5 +++++ ui/litellm-dashboard/out/index.html | 2 +- ui/litellm-dashboard/out/index.txt | 4 ++-- 32 files changed, 26 insertions(+), 26 deletions(-) rename litellm/proxy/_experimental/out/_next/static/{K8KXTbmuI2ArWjjdMi2iq => 84BZ5uERcn4DsO4_POsLl}/_buildManifest.js (100%) rename litellm/proxy/_experimental/out/_next/static/{K8KXTbmuI2ArWjjdMi2iq => 84BZ5uERcn4DsO4_POsLl}/_ssgManifest.js (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{2f6dbc85-17d29013b8ff3da5.js => 2f6dbc85-052c4579f80d66ae.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{69-e49705773ae41779.js => 69-04708d7d4a17c1ee.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/{566-ccd699ab19124658.js => 884-7576ee407a2ecbe6.js} (64%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/{_not-found-a166f65a6af292d9.js => _not-found-b1ee1381b72386c2.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/app/{layout-94ae1345f5d85446.js => layout-a756276e228ca058.js} (100%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/page-c804e862b63be987.js create mode 100644 litellm/proxy/_experimental/out/_next/static/chunks/app/page-e6190351ac8da62a.js rename litellm/proxy/_experimental/out/_next/static/chunks/{fd9d1056-dafd44dfa2da140c.js => fd9d1056-f960ab1e6d32b002.js} (100%) rename ui/litellm-dashboard/out/_next/static/chunks/webpack-5b257e1ab47d4b4a.js => litellm/proxy/_experimental/out/_next/static/chunks/webpack-de9c0fadf6a94b3b.js (98%) delete mode 100644 litellm/proxy/_experimental/out/_next/static/css/a1602eb39f799143.css create mode 100644 litellm/proxy/_experimental/out/_next/static/css/f04e46b02318b660.css rename ui/litellm-dashboard/out/_next/static/{K8KXTbmuI2ArWjjdMi2iq => 84BZ5uERcn4DsO4_POsLl}/_buildManifest.js (100%) rename ui/litellm-dashboard/out/_next/static/{K8KXTbmuI2ArWjjdMi2iq => 84BZ5uERcn4DsO4_POsLl}/_ssgManifest.js (100%) rename ui/litellm-dashboard/out/_next/static/chunks/{2f6dbc85-17d29013b8ff3da5.js => 2f6dbc85-052c4579f80d66ae.js} (100%) rename ui/litellm-dashboard/out/_next/static/chunks/{69-e49705773ae41779.js => 69-04708d7d4a17c1ee.js} (100%) rename ui/litellm-dashboard/out/_next/static/chunks/{566-ccd699ab19124658.js => 884-7576ee407a2ecbe6.js} (64%) rename ui/litellm-dashboard/out/_next/static/chunks/app/{_not-found-a166f65a6af292d9.js => _not-found-b1ee1381b72386c2.js} (100%) rename ui/litellm-dashboard/out/_next/static/chunks/app/{layout-94ae1345f5d85446.js => layout-a756276e228ca058.js} (100%) delete mode 100644 ui/litellm-dashboard/out/_next/static/chunks/app/page-c804e862b63be987.js create mode 100644 ui/litellm-dashboard/out/_next/static/chunks/app/page-e6190351ac8da62a.js rename ui/litellm-dashboard/out/_next/static/chunks/{fd9d1056-dafd44dfa2da140c.js => fd9d1056-f960ab1e6d32b002.js} (100%) rename litellm/proxy/_experimental/out/_next/static/chunks/webpack-5b257e1ab47d4b4a.js => ui/litellm-dashboard/out/_next/static/chunks/webpack-de9c0fadf6a94b3b.js (98%) delete mode 100644 ui/litellm-dashboard/out/_next/static/css/a1602eb39f799143.css create mode 100644 ui/litellm-dashboard/out/_next/static/css/f04e46b02318b660.css diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index 448d7cf877c..eaf57013553 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/K8KXTbmuI2ArWjjdMi2iq/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/84BZ5uERcn4DsO4_POsLl/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/K8KXTbmuI2ArWjjdMi2iq/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/84BZ5uERcn4DsO4_POsLl/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/K8KXTbmuI2ArWjjdMi2iq/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/84BZ5uERcn4DsO4_POsLl/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/K8KXTbmuI2ArWjjdMi2iq/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/84BZ5uERcn4DsO4_POsLl/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/2f6dbc85-17d29013b8ff3da5.js b/litellm/proxy/_experimental/out/_next/static/chunks/2f6dbc85-052c4579f80d66ae.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/chunks/2f6dbc85-17d29013b8ff3da5.js rename to litellm/proxy/_experimental/out/_next/static/chunks/2f6dbc85-052c4579f80d66ae.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/69-e49705773ae41779.js b/litellm/proxy/_experimental/out/_next/static/chunks/69-04708d7d4a17c1ee.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/chunks/69-e49705773ae41779.js rename to litellm/proxy/_experimental/out/_next/static/chunks/69-04708d7d4a17c1ee.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/566-ccd699ab19124658.js b/litellm/proxy/_experimental/out/_next/static/chunks/884-7576ee407a2ecbe6.js similarity index 64% rename from litellm/proxy/_experimental/out/_next/static/chunks/566-ccd699ab19124658.js rename to litellm/proxy/_experimental/out/_next/static/chunks/884-7576ee407a2ecbe6.js index 3b819d4158a..f0f47e76abf 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/566-ccd699ab19124658.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/884-7576ee407a2ecbe6.js @@ -1,4 +1,4 @@ -"use strict";(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[566],{12215:function(e,t,n){n.d(t,{iN:function(){return h},R_:function(){return d},EV:function(){return g},ez:function(){return p}});var r=n(41785),o=n(76991),a=[{index:7,opacity:.15},{index:6,opacity:.25},{index:5,opacity:.3},{index:5,opacity:.45},{index:5,opacity:.65},{index:5,opacity:.85},{index:4,opacity:.9},{index:3,opacity:.95},{index:2,opacity:.97},{index:1,opacity:.98}];function i(e){var t=e.r,n=e.g,o=e.b,a=(0,r.py)(t,n,o);return{h:360*a.h,s:a.s,v:a.v}}function l(e){var t=e.r,n=e.g,o=e.b;return"#".concat((0,r.vq)(t,n,o,!1))}function s(e,t,n){var r;return(r=Math.round(e.h)>=60&&240>=Math.round(e.h)?n?Math.round(e.h)-2*t:Math.round(e.h)+2*t:n?Math.round(e.h)+2*t:Math.round(e.h)-2*t)<0?r+=360:r>=360&&(r-=360),r}function c(e,t,n){var r;return 0===e.h&&0===e.s?e.s:((r=n?e.s-.16*t:4===t?e.s+.16:e.s+.05*t)>1&&(r=1),n&&5===t&&r>.1&&(r=.1),r<.06&&(r=.06),Number(r.toFixed(2)))}function u(e,t,n){var r;return(r=n?e.v+.05*t:e.v-.15*t)>1&&(r=1),Number(r.toFixed(2))}function d(e){for(var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=[],r=(0,o.uA)(e),d=5;d>0;d-=1){var p=i(r),f=l((0,o.uA)({h:s(p,d,!0),s:c(p,d,!0),v:u(p,d,!0)}));n.push(f)}n.push(l(r));for(var m=1;m<=4;m+=1){var g=i(r),h=l((0,o.uA)({h:s(g,m),s:c(g,m),v:u(g,m)}));n.push(h)}return"dark"===t.theme?a.map(function(e){var r,a,i,s=e.index,c=e.opacity;return l((r=(0,o.uA)(t.backgroundColor||"#141414"),a=(0,o.uA)(n[s]),i=100*c/100,{r:(a.r-r.r)*i+r.r,g:(a.g-r.g)*i+r.g,b:(a.b-r.b)*i+r.b}))}):n}var p={red:"#F5222D",volcano:"#FA541C",orange:"#FA8C16",gold:"#FAAD14",yellow:"#FADB14",lime:"#A0D911",green:"#52C41A",cyan:"#13C2C2",blue:"#1677FF",geekblue:"#2F54EB",purple:"#722ED1",magenta:"#EB2F96",grey:"#666666"},f={},m={};Object.keys(p).forEach(function(e){f[e]=d(p[e]),f[e].primary=f[e][5],m[e]=d(p[e],{theme:"dark",backgroundColor:"#141414"}),m[e].primary=m[e][5]}),f.red,f.volcano;var g=f.gold;f.orange,f.yellow,f.lime,f.green,f.cyan;var h=f.blue;f.geekblue,f.purple,f.magenta,f.grey,f.grey},8985:function(e,t,n){n.d(t,{E4:function(){return ej},jG:function(){return A},ks:function(){return Z},bf:function(){return F},CI:function(){return eD},fp:function(){return X},xy:function(){return eM}});var r,o,a=n(50833),i=n(80406),l=n(63787),s=n(5239),c=function(e){for(var t,n=0,r=0,o=e.length;o>=4;++r,o-=4)t=(65535&(t=255&e.charCodeAt(r)|(255&e.charCodeAt(++r))<<8|(255&e.charCodeAt(++r))<<16|(255&e.charCodeAt(++r))<<24))*1540483477+((t>>>16)*59797<<16),t^=t>>>24,n=(65535&t)*1540483477+((t>>>16)*59797<<16)^(65535&n)*1540483477+((n>>>16)*59797<<16);switch(o){case 3:n^=(255&e.charCodeAt(r+2))<<16;case 2:n^=(255&e.charCodeAt(r+1))<<8;case 1:n^=255&e.charCodeAt(r),n=(65535&n)*1540483477+((n>>>16)*59797<<16)}return n^=n>>>13,(((n=(65535&n)*1540483477+((n>>>16)*59797<<16))^n>>>15)>>>0).toString(36)},u=n(24050),d=n(64090),p=n.t(d,2);n(61475),n(92536);var f=n(47365),m=n(65127);function g(e){return e.join("%")}var h=function(){function e(t){(0,f.Z)(this,e),(0,a.Z)(this,"instanceId",void 0),(0,a.Z)(this,"cache",new Map),this.instanceId=t}return(0,m.Z)(e,[{key:"get",value:function(e){return this.opGet(g(e))}},{key:"opGet",value:function(e){return this.cache.get(e)||null}},{key:"update",value:function(e,t){return this.opUpdate(g(e),t)}},{key:"opUpdate",value:function(e,t){var n=t(this.cache.get(e));null===n?this.cache.delete(e):this.cache.set(e,n)}}]),e}(),b="data-token-hash",v="data-css-hash",y="__cssinjs_instance__",E=d.createContext({hashPriority:"low",cache:function(){var e=Math.random().toString(12).slice(2);if("undefined"!=typeof document&&document.head&&document.body){var t=document.body.querySelectorAll("style[".concat(v,"]"))||[],n=document.head.firstChild;Array.from(t).forEach(function(t){t[y]=t[y]||e,t[y]===e&&document.head.insertBefore(t,n)});var r={};Array.from(document.querySelectorAll("style[".concat(v,"]"))).forEach(function(t){var n,o=t.getAttribute(v);r[o]?t[y]===e&&(null===(n=t.parentNode)||void 0===n||n.removeChild(t)):r[o]=!0})}return new h(e)}(),defaultCache:!0}),w=n(6976),S=n(22127),x=function(){function e(){(0,f.Z)(this,e),(0,a.Z)(this,"cache",void 0),(0,a.Z)(this,"keys",void 0),(0,a.Z)(this,"cacheCallTimes",void 0),this.cache=new Map,this.keys=[],this.cacheCallTimes=0}return(0,m.Z)(e,[{key:"size",value:function(){return this.keys.length}},{key:"internalGet",value:function(e){var t,n,r=arguments.length>1&&void 0!==arguments[1]&&arguments[1],o={map:this.cache};return e.forEach(function(e){if(o){var t;o=null===(t=o)||void 0===t||null===(t=t.map)||void 0===t?void 0:t.get(e)}else o=void 0}),null!==(t=o)&&void 0!==t&&t.value&&r&&(o.value[1]=this.cacheCallTimes++),null===(n=o)||void 0===n?void 0:n.value}},{key:"get",value:function(e){var t;return null===(t=this.internalGet(e,!0))||void 0===t?void 0:t[0]}},{key:"has",value:function(e){return!!this.internalGet(e)}},{key:"set",value:function(t,n){var r=this;if(!this.has(t)){if(this.size()+1>e.MAX_CACHE_SIZE+e.MAX_CACHE_OFFSET){var o=this.keys.reduce(function(e,t){var n=(0,i.Z)(e,2)[1];return r.internalGet(t)[1]0,"[Ant Design CSS-in-JS] Theme should have at least one derivative function."),k+=1}return(0,m.Z)(e,[{key:"getDerivativeToken",value:function(e){return this.derivatives.reduce(function(t,n){return n(e,t)},void 0)}}]),e}(),T=new x;function A(e){var t=Array.isArray(e)?e:[e];return T.has(t)||T.set(t,new C(t)),T.get(t)}var N=new WeakMap,I={},R=new WeakMap;function _(e){var t=R.get(e)||"";return t||(Object.keys(e).forEach(function(n){var r=e[n];t+=n,r instanceof C?t+=r.id:r&&"object"===(0,w.Z)(r)?t+=_(r):t+=r}),R.set(e,t)),t}function P(e,t){return c("".concat(t,"_").concat(_(e)))}var M="random-".concat(Date.now(),"-").concat(Math.random()).replace(/\./g,""),L="_bAmBoO_",D=void 0,j=(0,S.Z)();function F(e){return"number"==typeof e?"".concat(e,"px"):e}function B(e,t,n){var r,o=arguments.length>3&&void 0!==arguments[3]?arguments[3]:{},i=arguments.length>4&&void 0!==arguments[4]&&arguments[4];if(i)return e;var l=(0,s.Z)((0,s.Z)({},o),{},(r={},(0,a.Z)(r,b,t),(0,a.Z)(r,v,n),r)),c=Object.keys(l).map(function(e){var t=l[e];return t?"".concat(e,'="').concat(t,'"'):null}).filter(function(e){return e}).join(" ");return"")}var Z=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"";return"--".concat(t?"".concat(t,"-"):"").concat(e).replace(/([a-z0-9])([A-Z])/g,"$1-$2").replace(/([A-Z]+)([A-Z][a-z0-9]+)/g,"$1-$2").replace(/([a-z])([A-Z0-9])/g,"$1-$2").toLowerCase()},U=function(e,t,n){var r,o={},a={};return Object.entries(e).forEach(function(e){var t=(0,i.Z)(e,2),r=t[0],l=t[1];if(null!=n&&null!==(s=n.preserve)&&void 0!==s&&s[r])a[r]=l;else if(("string"==typeof l||"number"==typeof l)&&!(null!=n&&null!==(c=n.ignore)&&void 0!==c&&c[r])){var s,c,u,d=Z(r,null==n?void 0:n.prefix);o[d]="number"!=typeof l||null!=n&&null!==(u=n.unitless)&&void 0!==u&&u[r]?String(l):"".concat(l,"px"),a[r]="var(".concat(d,")")}}),[a,(r={scope:null==n?void 0:n.scope},Object.keys(o).length?".".concat(t).concat(null!=r&&r.scope?".".concat(r.scope):"","{").concat(Object.entries(o).map(function(e){var t=(0,i.Z)(e,2),n=t[0],r=t[1];return"".concat(n,":").concat(r,";")}).join(""),"}"):"")]},z=n(24800),H=(0,s.Z)({},p).useInsertionEffect,G=H?function(e,t,n){return H(function(){return e(),t()},n)}:function(e,t,n){d.useMemo(e,n),(0,z.Z)(function(){return t(!0)},n)},W=void 0!==(0,s.Z)({},p).useInsertionEffect?function(e){var t=[],n=!1;return d.useEffect(function(){return n=!1,function(){n=!0,t.length&&t.forEach(function(e){return e()})}},e),function(e){n||t.push(e)}}:function(){return function(e){e()}};function $(e,t,n,r,o){var a=d.useContext(E).cache,s=g([e].concat((0,l.Z)(t))),c=W([s]),u=function(e){a.opUpdate(s,function(t){var r=(0,i.Z)(t||[void 0,void 0],2),o=r[0],a=[void 0===o?0:o,r[1]||n()];return e?e(a):a})};d.useMemo(function(){u()},[s]);var p=a.opGet(s)[1];return G(function(){null==o||o(p)},function(e){return u(function(t){var n=(0,i.Z)(t,2),r=n[0],a=n[1];return e&&0===r&&(null==o||o(p)),[r+1,a]}),function(){a.opUpdate(s,function(t){var n=(0,i.Z)(t||[],2),o=n[0],l=void 0===o?0:o,u=n[1];return 0==l-1?(c(function(){(e||!a.opGet(s))&&(null==r||r(u,!1))}),null):[l-1,u]})}},[s]),p}var V={},q=new Map,Y=function(e,t,n,r){var o=n.getDerivativeToken(e),a=(0,s.Z)((0,s.Z)({},o),t);return r&&(a=r(a)),a},K="token";function X(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{},r=(0,d.useContext)(E),o=r.cache.instanceId,a=r.container,p=n.salt,f=void 0===p?"":p,m=n.override,g=void 0===m?V:m,h=n.formatToken,w=n.getComputedToken,S=n.cssVar,x=function(e,t){for(var n=N,r=0;r=(q.get(e)||0)}),n.length-r.length>0&&r.forEach(function(e){"undefined"!=typeof document&&document.querySelectorAll("style[".concat(b,'="').concat(e,'"]')).forEach(function(e){if(e[y]===o){var t;null===(t=e.parentNode)||void 0===t||t.removeChild(e)}}),q.delete(e)})},function(e){var t=(0,i.Z)(e,4),n=t[0],r=t[3];if(S&&r){var l=(0,u.hq)(r,c("css-variables-".concat(n._themeKey)),{mark:v,prepend:"queue",attachTo:a,priority:-999});l[y]=o,l.setAttribute(b,n._themeKey)}})}var Q=n(14749),J={animationIterationCount:1,borderImageOutset:1,borderImageSlice:1,borderImageWidth:1,boxFlex:1,boxFlexGroup:1,boxOrdinalGroup:1,columnCount:1,columns:1,flex:1,flexGrow:1,flexPositive:1,flexShrink:1,flexNegative:1,flexOrder:1,gridRow:1,gridRowEnd:1,gridRowSpan:1,gridRowStart:1,gridColumn:1,gridColumnEnd:1,gridColumnSpan:1,gridColumnStart:1,msGridRow:1,msGridRowSpan:1,msGridColumn:1,msGridColumnSpan:1,fontWeight:1,lineHeight:1,opacity:1,order:1,orphans:1,tabSize:1,widows:1,zIndex:1,zoom:1,WebkitLineClamp:1,fillOpacity:1,floodOpacity:1,stopOpacity:1,strokeDasharray:1,strokeDashoffset:1,strokeMiterlimit:1,strokeOpacity:1,strokeWidth:1},ee="comm",et="rule",en="decl",er=Math.abs,eo=String.fromCharCode;function ea(e,t,n){return e.replace(t,n)}function ei(e,t){return 0|e.charCodeAt(t)}function el(e,t,n){return e.slice(t,n)}function es(e){return e.length}function ec(e,t){return t.push(e),e}function eu(e,t){for(var n="",r=0;r0?f[v]+" "+y:ea(y,/&\f/g,f[v])).trim())&&(s[b++]=E);return ev(e,t,n,0===o?et:l,s,c,u,d)}function eO(e,t,n,r,o){return ev(e,t,n,en,el(e,0,r),el(e,r+1,-1),r,o)}var ek="data-ant-cssinjs-cache-path",eC="_FILE_STYLE__",eT=!0,eA="_multi_value_";function eN(e){var t,n,r;return eu((r=function e(t,n,r,o,a,i,l,s,c){for(var u,d,p,f=0,m=0,g=l,h=0,b=0,v=0,y=1,E=1,w=1,S=0,x="",O=a,k=i,C=o,T=x;E;)switch(v=S,S=ey()){case 40:if(108!=v&&58==ei(T,g-1)){-1!=(d=T+=ea(eS(S),"&","&\f"),p=er(f?s[f-1]:0),d.indexOf("&\f",p))&&(w=-1);break}case 34:case 39:case 91:T+=eS(S);break;case 9:case 10:case 13:case 32:T+=function(e){for(;eh=eE();)if(eh<33)ey();else break;return ew(e)>2||ew(eh)>3?"":" "}(v);break;case 92:T+=function(e,t){for(var n;--t&&ey()&&!(eh<48)&&!(eh>102)&&(!(eh>57)||!(eh<65))&&(!(eh>70)||!(eh<97)););return n=eg+(t<6&&32==eE()&&32==ey()),el(eb,e,n)}(eg-1,7);continue;case 47:switch(eE()){case 42:case 47:ec(ev(u=function(e,t){for(;ey();)if(e+eh===57)break;else if(e+eh===84&&47===eE())break;return"/*"+el(eb,t,eg-1)+"*"+eo(47===e?e:ey())}(ey(),eg),n,r,ee,eo(eh),el(u,2,-2),0,c),c);break;default:T+="/"}break;case 123*y:s[f++]=es(T)*w;case 125*y:case 59:case 0:switch(S){case 0:case 125:E=0;case 59+m:-1==w&&(T=ea(T,/\f/g,"")),b>0&&es(T)-g&&ec(b>32?eO(T+";",o,r,g-1,c):eO(ea(T," ","")+";",o,r,g-2,c),c);break;case 59:T+=";";default:if(ec(C=ex(T,n,r,f,m,a,s,x,O=[],k=[],g,i),i),123===S){if(0===m)e(T,n,C,C,O,i,g,s,k);else switch(99===h&&110===ei(T,3)?100:h){case 100:case 108:case 109:case 115:e(t,C,C,o&&ec(ex(t,C,C,0,0,a,s,x,a,O=[],g,k),k),a,k,g,s,o?O:k);break;default:e(T,C,C,C,[""],k,0,s,k)}}}f=m=b=0,y=w=1,x=T="",g=l;break;case 58:g=1+es(T),b=v;default:if(y<1){if(123==S)--y;else if(125==S&&0==y++&&125==(eh=eg>0?ei(eb,--eg):0,ef--,10===eh&&(ef=1,ep--),eh))continue}switch(T+=eo(S),S*y){case 38:w=m>0?1:(T+="\f",-1);break;case 44:s[f++]=(es(T)-1)*w,w=1;break;case 64:45===eE()&&(T+=eS(ey())),h=eE(),m=g=es(x=T+=function(e){for(;!ew(eE());)ey();return el(eb,e,eg)}(eg)),S++;break;case 45:45===v&&2==es(T)&&(y=0)}}return i}("",null,null,null,[""],(n=t=e,ep=ef=1,em=es(eb=n),eg=0,t=[]),0,[0],t),eb="",r),ed).replace(/\{%%%\:[^;];}/g,";")}var eI=function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{root:!0,parentSelectors:[]},o=r.root,a=r.injectHash,c=r.parentSelectors,d=n.hashId,p=n.layer,f=(n.path,n.hashPriority),m=n.transformers,g=void 0===m?[]:m;n.linters;var h="",b={};function v(t){var r=t.getName(d);if(!b[r]){var o=e(t.style,n,{root:!1,parentSelectors:c}),a=(0,i.Z)(o,1)[0];b[r]="@keyframes ".concat(t.getName(d)).concat(a)}}if((function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[];return t.forEach(function(t){Array.isArray(t)?e(t,n):t&&n.push(t)}),n})(Array.isArray(t)?t:[t]).forEach(function(t){var r="string"!=typeof t||o?t:{};if("string"==typeof r)h+="".concat(r,"\n");else if(r._keyframe)v(r);else{var u=g.reduce(function(e,t){var n;return(null==t||null===(n=t.visit)||void 0===n?void 0:n.call(t,e))||e},r);Object.keys(u).forEach(function(t){var r=u[t];if("object"!==(0,w.Z)(r)||!r||"animationName"===t&&r._keyframe||"object"===(0,w.Z)(r)&&r&&("_skip_check_"in r||eA in r)){function p(e,t){var n=e.replace(/[A-Z]/g,function(e){return"-".concat(e.toLowerCase())}),r=t;J[e]||"number"!=typeof r||0===r||(r="".concat(r,"px")),"animationName"===e&&null!=t&&t._keyframe&&(v(t),r=t.getName(d)),h+="".concat(n,":").concat(r,";")}var m,g=null!==(m=null==r?void 0:r.value)&&void 0!==m?m:r;"object"===(0,w.Z)(r)&&null!=r&&r[eA]&&Array.isArray(g)?g.forEach(function(e){p(t,e)}):p(t,g)}else{var y=!1,E=t.trim(),S=!1;(o||a)&&d?E.startsWith("@")?y=!0:E=function(e,t,n){if(!t)return e;var r=".".concat(t),o="low"===n?":where(".concat(r,")"):r;return e.split(",").map(function(e){var t,n=e.trim().split(/\s+/),r=n[0]||"",a=(null===(t=r.match(/^\w+/))||void 0===t?void 0:t[0])||"";return[r="".concat(a).concat(o).concat(r.slice(a.length))].concat((0,l.Z)(n.slice(1))).join(" ")}).join(",")}(t,d,f):o&&!d&&("&"===E||""===E)&&(E="",S=!0);var x=e(r,n,{root:S,injectHash:y,parentSelectors:[].concat((0,l.Z)(c),[E])}),O=(0,i.Z)(x,2),k=O[0],C=O[1];b=(0,s.Z)((0,s.Z)({},b),C),h+="".concat(E).concat(k)}})}}),o){if(p&&(void 0===D&&(D=function(e,t,n){if((0,S.Z)()){(0,u.hq)(e,M);var r,o,a=document.createElement("div");a.style.position="fixed",a.style.left="0",a.style.top="0",null==t||t(a),document.body.appendChild(a);var i=n?n(a):null===(r=getComputedStyle(a).content)||void 0===r?void 0:r.includes(L);return null===(o=a.parentNode)||void 0===o||o.removeChild(a),(0,u.jL)(M),i}return!1}("@layer ".concat(M," { .").concat(M,' { content: "').concat(L,'"!important; } }'),function(e){e.className=M})),D)){var y=p.split(","),E=y[y.length-1].trim();h="@layer ".concat(E," {").concat(h,"}"),y.length>1&&(h="@layer ".concat(p,"{%%%:%}").concat(h))}}else h="{".concat(h,"}");return[h,b]};function eR(e,t){return c("".concat(e.join("%")).concat(t))}function e_(){return null}var eP="style";function eM(e,t){var n=e.token,o=e.path,s=e.hashId,c=e.layer,p=e.nonce,f=e.clientOnly,m=e.order,g=void 0===m?0:m,h=d.useContext(E),w=h.autoClear,x=(h.mock,h.defaultCache),O=h.hashPriority,k=h.container,C=h.ssrInline,T=h.transformers,A=h.linters,N=h.cache,I=n._tokenKey,R=[I].concat((0,l.Z)(o)),_=$(eP,R,function(){var e=R.join("|");if(!function(){if(!r&&(r={},(0,S.Z)())){var e,t=document.createElement("div");t.className=ek,t.style.position="fixed",t.style.visibility="hidden",t.style.top="-9999px",document.body.appendChild(t);var n=getComputedStyle(t).content||"";(n=n.replace(/^"/,"").replace(/"$/,"")).split(";").forEach(function(e){var t=e.split(":"),n=(0,i.Z)(t,2),o=n[0],a=n[1];r[o]=a});var o=document.querySelector("style[".concat(ek,"]"));o&&(eT=!1,null===(e=o.parentNode)||void 0===e||e.removeChild(o)),document.body.removeChild(t)}}(),r[e]){var n=function(e){var t=r[e],n=null;if(t&&(0,S.Z)()){if(eT)n=eC;else{var o=document.querySelector("style[".concat(v,'="').concat(r[e],'"]'));o?n=o.innerHTML:delete r[e]}}return[n,t]}(e),a=(0,i.Z)(n,2),l=a[0],u=a[1];if(l)return[l,I,u,{},f,g]}var d=eI(t(),{hashId:s,hashPriority:O,layer:c,path:o.join("-"),transformers:T,linters:A}),p=(0,i.Z)(d,2),m=p[0],h=p[1],b=eN(m),y=eR(R,b);return[b,I,y,h,f,g]},function(e,t){var n=(0,i.Z)(e,3)[2];(t||w)&&j&&(0,u.jL)(n,{mark:v})},function(e){var t=(0,i.Z)(e,4),n=t[0],r=(t[1],t[2]),o=t[3];if(j&&n!==eC){var a={mark:v,prepend:"queue",attachTo:k,priority:g},l="function"==typeof p?p():p;l&&(a.csp={nonce:l});var s=(0,u.hq)(n,r,a);s[y]=N.instanceId,s.setAttribute(b,I),Object.keys(o).forEach(function(e){(0,u.hq)(eN(o[e]),"_effect-".concat(e),a)})}}),P=(0,i.Z)(_,3),M=P[0],L=P[1],D=P[2];return function(e){var t,n;return t=C&&!j&&x?d.createElement("style",(0,Q.Z)({},(n={},(0,a.Z)(n,b,L),(0,a.Z)(n,v,D),n),{dangerouslySetInnerHTML:{__html:M}})):d.createElement(e_,null),d.createElement(d.Fragment,null,t,e)}}var eL="cssVar",eD=function(e,t){var n=e.key,r=e.prefix,o=e.unitless,a=e.ignore,s=e.token,c=e.scope,p=void 0===c?"":c,f=(0,d.useContext)(E),m=f.cache.instanceId,g=f.container,h=s._tokenKey,w=[].concat((0,l.Z)(e.path),[n,p,h]);return $(eL,w,function(){var e=U(t(),n,{prefix:r,unitless:o,ignore:a,scope:p}),l=(0,i.Z)(e,2),s=l[0],c=l[1],u=eR(w,c);return[s,c,u,n]},function(e){var t=(0,i.Z)(e,3)[2];j&&(0,u.jL)(t,{mark:v})},function(e){var t=(0,i.Z)(e,3),r=t[1],o=t[2];if(r){var a=(0,u.hq)(r,o,{mark:v,prepend:"queue",attachTo:g,priority:-999});a[y]=m,a.setAttribute(b,n)}})};o={},(0,a.Z)(o,eP,function(e,t,n){var r=(0,i.Z)(e,6),o=r[0],a=r[1],l=r[2],s=r[3],c=r[4],u=r[5],d=(n||{}).plain;if(c)return null;var p=o,f={"data-rc-order":"prependQueue","data-rc-priority":"".concat(u)};return p=B(o,a,l,f,d),s&&Object.keys(s).forEach(function(e){if(!t[e]){t[e]=!0;var n=eN(s[e]);p+=B(n,a,"_effect-".concat(e),f,d)}}),[u,l,p]}),(0,a.Z)(o,K,function(e,t,n){var r=(0,i.Z)(e,5),o=r[2],a=r[3],l=r[4],s=(n||{}).plain;if(!a)return null;var c=o._tokenKey,u=B(a,l,c,{"data-rc-order":"prependQueue","data-rc-priority":"".concat(-999)},s);return[-999,c,u]}),(0,a.Z)(o,eL,function(e,t,n){var r=(0,i.Z)(e,4),o=r[1],a=r[2],l=r[3],s=(n||{}).plain;if(!o)return null;var c=B(o,l,a,{"data-rc-order":"prependQueue","data-rc-priority":"".concat(-999)},s);return[-999,a,c]});var ej=function(){function e(t,n){(0,f.Z)(this,e),(0,a.Z)(this,"name",void 0),(0,a.Z)(this,"style",void 0),(0,a.Z)(this,"_keyframe",!0),this.name=t,this.style=n}return(0,m.Z)(e,[{key:"getName",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"";return e?"".concat(e,"-").concat(this.name):this.name}}]),e}();function eF(e){return e.notSplit=!0,e}eF(["borderTop","borderBottom"]),eF(["borderTop"]),eF(["borderBottom"]),eF(["borderLeft","borderRight"]),eF(["borderLeft"]),eF(["borderRight"])},60688:function(e,t,n){n.d(t,{Z:function(){return A}});var r=n(14749),o=n(80406),a=n(50833),i=n(60635),l=n(64090),s=n(16480),c=n.n(s),u=n(12215),d=n(67689),p=n(5239),f=n(6976),m=n(24050),g=n(74687),h=n(53850);function b(e){return"object"===(0,f.Z)(e)&&"string"==typeof e.name&&"string"==typeof e.theme&&("object"===(0,f.Z)(e.icon)||"function"==typeof e.icon)}function v(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return Object.keys(e).reduce(function(t,n){var r=e[n];return"class"===n?(t.className=r,delete t.class):(delete t[n],t[n.replace(/-(.)/g,function(e,t){return t.toUpperCase()})]=r),t},{})}function y(e){return(0,u.R_)(e)[0]}function E(e){return e?Array.isArray(e)?e:[e]:[]}var w=function(e){var t=(0,l.useContext)(d.Z),n=t.csp,r=t.prefixCls,o="\n.anticon {\n display: inline-block;\n color: inherit;\n font-style: normal;\n line-height: 0;\n text-align: center;\n text-transform: none;\n vertical-align: -0.125em;\n text-rendering: optimizeLegibility;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale;\n}\n\n.anticon > * {\n line-height: 1;\n}\n\n.anticon svg {\n display: inline-block;\n}\n\n.anticon::before {\n display: none;\n}\n\n.anticon .anticon-icon {\n display: block;\n}\n\n.anticon[tabindex] {\n cursor: pointer;\n}\n\n.anticon-spin::before,\n.anticon-spin {\n display: inline-block;\n -webkit-animation: loadingCircle 1s infinite linear;\n animation: loadingCircle 1s infinite linear;\n}\n\n@-webkit-keyframes loadingCircle {\n 100% {\n -webkit-transform: rotate(360deg);\n transform: rotate(360deg);\n }\n}\n\n@keyframes loadingCircle {\n 100% {\n -webkit-transform: rotate(360deg);\n transform: rotate(360deg);\n }\n}\n";r&&(o=o.replace(/anticon/g,r)),(0,l.useEffect)(function(){var t=e.current,r=(0,g.A)(t);(0,m.hq)(o,"@ant-design-icons",{prepend:!0,csp:n,attachTo:r})},[])},S=["icon","className","onClick","style","primaryColor","secondaryColor"],x={primaryColor:"#333",secondaryColor:"#E6E6E6",calculated:!1},O=function(e){var t,n,r=e.icon,o=e.className,a=e.onClick,s=e.style,c=e.primaryColor,u=e.secondaryColor,d=(0,i.Z)(e,S),f=l.useRef(),m=x;if(c&&(m={primaryColor:c,secondaryColor:u||y(c)}),w(f),t=b(r),n="icon should be icon definiton, but got ".concat(r),(0,h.ZP)(t,"[@ant-design/icons] ".concat(n)),!b(r))return null;var g=r;return g&&"function"==typeof g.icon&&(g=(0,p.Z)((0,p.Z)({},g),{},{icon:g.icon(m.primaryColor,m.secondaryColor)})),function e(t,n,r){return r?l.createElement(t.tag,(0,p.Z)((0,p.Z)({key:n},v(t.attrs)),r),(t.children||[]).map(function(r,o){return e(r,"".concat(n,"-").concat(t.tag,"-").concat(o))})):l.createElement(t.tag,(0,p.Z)({key:n},v(t.attrs)),(t.children||[]).map(function(r,o){return e(r,"".concat(n,"-").concat(t.tag,"-").concat(o))}))}(g.icon,"svg-".concat(g.name),(0,p.Z)((0,p.Z)({className:o,onClick:a,style:s,"data-icon":g.name,width:"1em",height:"1em",fill:"currentColor","aria-hidden":"true"},d),{},{ref:f}))};function k(e){var t=E(e),n=(0,o.Z)(t,2),r=n[0],a=n[1];return O.setTwoToneColors({primaryColor:r,secondaryColor:a})}O.displayName="IconReact",O.getTwoToneColors=function(){return(0,p.Z)({},x)},O.setTwoToneColors=function(e){var t=e.primaryColor,n=e.secondaryColor;x.primaryColor=t,x.secondaryColor=n||y(t),x.calculated=!!n};var C=["className","icon","spin","rotate","tabIndex","onClick","twoToneColor"];k(u.iN.primary);var T=l.forwardRef(function(e,t){var n,s=e.className,u=e.icon,p=e.spin,f=e.rotate,m=e.tabIndex,g=e.onClick,h=e.twoToneColor,b=(0,i.Z)(e,C),v=l.useContext(d.Z),y=v.prefixCls,w=void 0===y?"anticon":y,S=v.rootClassName,x=c()(S,w,(n={},(0,a.Z)(n,"".concat(w,"-").concat(u.name),!!u.name),(0,a.Z)(n,"".concat(w,"-spin"),!!p||"loading"===u.name),n),s),k=m;void 0===k&&g&&(k=-1);var T=E(h),A=(0,o.Z)(T,2),N=A[0],I=A[1];return l.createElement("span",(0,r.Z)({role:"img","aria-label":u.name},b,{ref:t,tabIndex:k,onClick:g,className:x}),l.createElement(O,{icon:u,primaryColor:N,secondaryColor:I,style:f?{msTransform:"rotate(".concat(f,"deg)"),transform:"rotate(".concat(f,"deg)")}:void 0}))});T.displayName="AntdIcon",T.getTwoToneColor=function(){var e=O.getTwoToneColors();return e.calculated?[e.primaryColor,e.secondaryColor]:e.primaryColor},T.setTwoToneColor=k;var A=T},67689:function(e,t,n){var r=(0,n(64090).createContext)({});t.Z=r},99537:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm193.5 301.7l-210.6 292a31.8 31.8 0 01-51.7 0L318.5 484.9c-3.8-5.3 0-12.7 6.5-12.7h46.9c10.2 0 19.9 4.9 25.9 13.3l71.2 98.8 157.2-218c6-8.3 15.6-13.3 25.9-13.3H699c6.5 0 10.3 7.4 6.5 12.7z"}}]},name:"check-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},90507:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M912 190h-69.9c-9.8 0-19.1 4.5-25.1 12.2L404.7 724.5 207 474a32 32 0 00-25.1-12.2H112c-6.7 0-10.4 7.7-6.3 12.9l273.9 347c12.8 16.2 37.4 16.2 50.3 0l488.4-618.9c4.1-5.1.4-12.8-6.3-12.8z"}}]},name:"check",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},77136:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{"fill-rule":"evenodd",viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64c247.4 0 448 200.6 448 448S759.4 960 512 960 64 759.4 64 512 264.6 64 512 64zm127.98 274.82h-.04l-.08.06L512 466.75 384.14 338.88c-.04-.05-.06-.06-.08-.06a.12.12 0 00-.07 0c-.03 0-.05.01-.09.05l-45.02 45.02a.2.2 0 00-.05.09.12.12 0 000 .07v.02a.27.27 0 00.06.06L466.75 512 338.88 639.86c-.05.04-.06.06-.06.08a.12.12 0 000 .07c0 .03.01.05.05.09l45.02 45.02a.2.2 0 00.09.05.12.12 0 00.07 0c.02 0 .04-.01.08-.05L512 557.25l127.86 127.87c.04.04.06.05.08.05a.12.12 0 00.07 0c.03 0 .05-.01.09-.05l45.02-45.02a.2.2 0 00.05-.09.12.12 0 000-.07v-.02a.27.27 0 00-.05-.06L557.25 512l127.87-127.86c.04-.04.05-.06.05-.08a.12.12 0 000-.07c0-.03-.01-.05-.05-.09l-45.02-45.02a.2.2 0 00-.09-.05.12.12 0 00-.07 0z"}}]},name:"close-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},81303:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{"fill-rule":"evenodd",viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M799.86 166.31c.02 0 .04.02.08.06l57.69 57.7c.04.03.05.05.06.08a.12.12 0 010 .06c0 .03-.02.05-.06.09L569.93 512l287.7 287.7c.04.04.05.06.06.09a.12.12 0 010 .07c0 .02-.02.04-.06.08l-57.7 57.69c-.03.04-.05.05-.07.06a.12.12 0 01-.07 0c-.03 0-.05-.02-.09-.06L512 569.93l-287.7 287.7c-.04.04-.06.05-.09.06a.12.12 0 01-.07 0c-.02 0-.04-.02-.08-.06l-57.69-57.7c-.04-.03-.05-.05-.06-.07a.12.12 0 010-.07c0-.03.02-.05.06-.09L454.07 512l-287.7-287.7c-.04-.04-.05-.06-.06-.09a.12.12 0 010-.07c0-.02.02-.04.06-.08l57.7-57.69c.03-.04.05-.05.07-.06a.12.12 0 01.07 0c.03 0 .05.02.09.06L512 454.07l287.7-287.7c.04-.04.06-.05.09-.06a.12.12 0 01.07 0z"}}]},name:"close",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},20383:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M884 256h-75c-5.1 0-9.9 2.5-12.9 6.6L512 654.2 227.9 262.6c-3-4.1-7.8-6.6-12.9-6.6h-75c-6.5 0-10.3 7.4-6.5 12.7l352.6 486.1c12.8 17.6 39 17.6 51.7 0l352.6-486.1c3.9-5.3.1-12.7-6.4-12.7z"}}]},name:"down",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},31413:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M176 511a56 56 0 10112 0 56 56 0 10-112 0zm280 0a56 56 0 10112 0 56 56 0 10-112 0zm280 0a56 56 0 10112 0 56 56 0 10-112 0z"}}]},name:"ellipsis",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},20653:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm-32 232c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v272c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8V296zm32 440a48.01 48.01 0 010-96 48.01 48.01 0 010 96z"}}]},name:"exclamation-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},41311:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M942.2 486.2C847.4 286.5 704.1 186 512 186c-192.2 0-335.4 100.5-430.2 300.3a60.3 60.3 0 000 51.5C176.6 737.5 319.9 838 512 838c192.2 0 335.4-100.5 430.2-300.3 7.7-16.2 7.7-35 0-51.5zM512 766c-161.3 0-279.4-81.8-362.7-254C232.6 339.8 350.7 258 512 258c161.3 0 279.4 81.8 362.7 254C791.5 684.2 673.4 766 512 766zm-4-430c-97.2 0-176 78.8-176 176s78.8 176 176 176 176-78.8 176-176-78.8-176-176-176zm0 288c-61.9 0-112-50.1-112-112s50.1-112 112-112 112 50.1 112 112-50.1 112-112 112z"}}]},name:"eye",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},40388:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M512 64C264.6 64 64 264.6 64 512s200.6 448 448 448 448-200.6 448-448S759.4 64 512 64zm32 664c0 4.4-3.6 8-8 8h-48c-4.4 0-8-3.6-8-8V456c0-4.4 3.6-8 8-8h48c4.4 0 8 3.6 8 8v272zm-32-344a48.01 48.01 0 010-96 48.01 48.01 0 010 96z"}}]},name:"info-circle",theme:"filled"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},66155:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"0 0 1024 1024",focusable:"false"},children:[{tag:"path",attrs:{d:"M988 548c-19.9 0-36-16.1-36-36 0-59.4-11.6-117-34.6-171.3a440.45 440.45 0 00-94.3-139.9 437.71 437.71 0 00-139.9-94.3C629 83.6 571.4 72 512 72c-19.9 0-36-16.1-36-36s16.1-36 36-36c69.1 0 136.2 13.5 199.3 40.3C772.3 66 827 103 874 150c47 47 83.9 101.8 109.7 162.7 26.7 63.1 40.2 130.2 40.2 199.3.1 19.9-16 36-35.9 36z"}}]},name:"loading",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},50459:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M765.7 486.8L314.9 134.7A7.97 7.97 0 00302 141v77.3c0 4.9 2.3 9.6 6.1 12.6l360 281.1-360 281.1c-3.9 3-6.1 7.7-6.1 12.6V883c0 6.7 7.7 10.4 12.9 6.3l450.8-352.1a31.96 31.96 0 000-50.4z"}}]},name:"right",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},96871:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M909.6 854.5L649.9 594.8C690.2 542.7 712 479 712 412c0-80.2-31.3-155.4-87.9-212.1-56.6-56.7-132-87.9-212.1-87.9s-155.5 31.3-212.1 87.9C143.2 256.5 112 331.8 112 412c0 80.1 31.3 155.5 87.9 212.1C256.5 680.8 331.8 712 412 712c67 0 130.6-21.8 182.7-62l259.7 259.6a8.2 8.2 0 0011.6 0l43.6-43.5a8.2 8.2 0 000-11.6zM570.4 570.4C528 612.7 471.8 636 412 636s-116-23.3-158.4-65.6C211.3 528 188 471.8 188 412s23.3-116.1 65.6-158.4C296 211.3 352.2 188 412 188s116.1 23.2 158.4 65.6S636 352.2 636 412s-23.3 116.1-65.6 158.4z"}}]},name:"search",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},97766:function(e,t,n){n.d(t,{Z:function(){return l}});var r=n(14749),o=n(64090),a={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M400 317.7h73.9V656c0 4.4 3.6 8 8 8h60c4.4 0 8-3.6 8-8V317.7H624c6.7 0 10.4-7.7 6.3-12.9L518.3 163a8 8 0 00-12.6 0l-112 141.7c-4.1 5.3-.4 13 6.3 13zM878 626h-60c-4.4 0-8 3.6-8 8v154H214V634c0-4.4-3.6-8-8-8h-60c-4.4 0-8 3.6-8 8v198c0 17.7 14.3 32 32 32h684c17.7 0 32-14.3 32-32V634c0-4.4-3.6-8-8-8z"}}]},name:"upload",theme:"outlined"},i=n(60688),l=o.forwardRef(function(e,t){return o.createElement(i.Z,(0,r.Z)({},e,{ref:t,icon:a}))})},41785:function(e,t,n){n.d(t,{T6:function(){return p},VD:function(){return f},WE:function(){return c},Yt:function(){return m},lC:function(){return a},py:function(){return s},rW:function(){return o},s:function(){return d},ve:function(){return l},vq:function(){return u}});var r=n(27974);function o(e,t,n){return{r:255*(0,r.sh)(e,255),g:255*(0,r.sh)(t,255),b:255*(0,r.sh)(n,255)}}function a(e,t,n){var o=Math.max(e=(0,r.sh)(e,255),t=(0,r.sh)(t,255),n=(0,r.sh)(n,255)),a=Math.min(e,t,n),i=0,l=0,s=(o+a)/2;if(o===a)l=0,i=0;else{var c=o-a;switch(l=s>.5?c/(2-o-a):c/(o+a),o){case e:i=(t-n)/c+(t1&&(n-=1),n<1/6)?e+6*n*(t-e):n<.5?t:n<2/3?e+(t-e)*(2/3-n)*6:e}function l(e,t,n){if(e=(0,r.sh)(e,360),t=(0,r.sh)(t,100),n=(0,r.sh)(n,100),0===t)a=n,l=n,o=n;else{var o,a,l,s=n<.5?n*(1+t):n+t-n*t,c=2*n-s;o=i(c,s,e+1/3),a=i(c,s,e),l=i(c,s,e-1/3)}return{r:255*o,g:255*a,b:255*l}}function s(e,t,n){var o=Math.max(e=(0,r.sh)(e,255),t=(0,r.sh)(t,255),n=(0,r.sh)(n,255)),a=Math.min(e,t,n),i=0,l=o-a;if(o===a)i=0;else{switch(o){case e:i=(t-n)/l+(t>16,g:(65280&e)>>8,b:255&e}}},6564:function(e,t,n){n.d(t,{R:function(){return r}});var r={aliceblue:"#f0f8ff",antiquewhite:"#faebd7",aqua:"#00ffff",aquamarine:"#7fffd4",azure:"#f0ffff",beige:"#f5f5dc",bisque:"#ffe4c4",black:"#000000",blanchedalmond:"#ffebcd",blue:"#0000ff",blueviolet:"#8a2be2",brown:"#a52a2a",burlywood:"#deb887",cadetblue:"#5f9ea0",chartreuse:"#7fff00",chocolate:"#d2691e",coral:"#ff7f50",cornflowerblue:"#6495ed",cornsilk:"#fff8dc",crimson:"#dc143c",cyan:"#00ffff",darkblue:"#00008b",darkcyan:"#008b8b",darkgoldenrod:"#b8860b",darkgray:"#a9a9a9",darkgreen:"#006400",darkgrey:"#a9a9a9",darkkhaki:"#bdb76b",darkmagenta:"#8b008b",darkolivegreen:"#556b2f",darkorange:"#ff8c00",darkorchid:"#9932cc",darkred:"#8b0000",darksalmon:"#e9967a",darkseagreen:"#8fbc8f",darkslateblue:"#483d8b",darkslategray:"#2f4f4f",darkslategrey:"#2f4f4f",darkturquoise:"#00ced1",darkviolet:"#9400d3",deeppink:"#ff1493",deepskyblue:"#00bfff",dimgray:"#696969",dimgrey:"#696969",dodgerblue:"#1e90ff",firebrick:"#b22222",floralwhite:"#fffaf0",forestgreen:"#228b22",fuchsia:"#ff00ff",gainsboro:"#dcdcdc",ghostwhite:"#f8f8ff",goldenrod:"#daa520",gold:"#ffd700",gray:"#808080",green:"#008000",greenyellow:"#adff2f",grey:"#808080",honeydew:"#f0fff0",hotpink:"#ff69b4",indianred:"#cd5c5c",indigo:"#4b0082",ivory:"#fffff0",khaki:"#f0e68c",lavenderblush:"#fff0f5",lavender:"#e6e6fa",lawngreen:"#7cfc00",lemonchiffon:"#fffacd",lightblue:"#add8e6",lightcoral:"#f08080",lightcyan:"#e0ffff",lightgoldenrodyellow:"#fafad2",lightgray:"#d3d3d3",lightgreen:"#90ee90",lightgrey:"#d3d3d3",lightpink:"#ffb6c1",lightsalmon:"#ffa07a",lightseagreen:"#20b2aa",lightskyblue:"#87cefa",lightslategray:"#778899",lightslategrey:"#778899",lightsteelblue:"#b0c4de",lightyellow:"#ffffe0",lime:"#00ff00",limegreen:"#32cd32",linen:"#faf0e6",magenta:"#ff00ff",maroon:"#800000",mediumaquamarine:"#66cdaa",mediumblue:"#0000cd",mediumorchid:"#ba55d3",mediumpurple:"#9370db",mediumseagreen:"#3cb371",mediumslateblue:"#7b68ee",mediumspringgreen:"#00fa9a",mediumturquoise:"#48d1cc",mediumvioletred:"#c71585",midnightblue:"#191970",mintcream:"#f5fffa",mistyrose:"#ffe4e1",moccasin:"#ffe4b5",navajowhite:"#ffdead",navy:"#000080",oldlace:"#fdf5e6",olive:"#808000",olivedrab:"#6b8e23",orange:"#ffa500",orangered:"#ff4500",orchid:"#da70d6",palegoldenrod:"#eee8aa",palegreen:"#98fb98",paleturquoise:"#afeeee",palevioletred:"#db7093",papayawhip:"#ffefd5",peachpuff:"#ffdab9",peru:"#cd853f",pink:"#ffc0cb",plum:"#dda0dd",powderblue:"#b0e0e6",purple:"#800080",rebeccapurple:"#663399",red:"#ff0000",rosybrown:"#bc8f8f",royalblue:"#4169e1",saddlebrown:"#8b4513",salmon:"#fa8072",sandybrown:"#f4a460",seagreen:"#2e8b57",seashell:"#fff5ee",sienna:"#a0522d",silver:"#c0c0c0",skyblue:"#87ceeb",slateblue:"#6a5acd",slategray:"#708090",slategrey:"#708090",snow:"#fffafa",springgreen:"#00ff7f",steelblue:"#4682b4",tan:"#d2b48c",teal:"#008080",thistle:"#d8bfd8",tomato:"#ff6347",turquoise:"#40e0d0",violet:"#ee82ee",wheat:"#f5deb3",white:"#ffffff",whitesmoke:"#f5f5f5",yellow:"#ffff00",yellowgreen:"#9acd32"}},76991:function(e,t,n){n.d(t,{uA:function(){return i}});var r=n(41785),o=n(6564),a=n(27974);function i(e){var t={r:0,g:0,b:0},n=1,i=null,l=null,s=null,c=!1,p=!1;return"string"==typeof e&&(e=function(e){if(0===(e=e.trim().toLowerCase()).length)return!1;var t=!1;if(o.R[e])e=o.R[e],t=!0;else if("transparent"===e)return{r:0,g:0,b:0,a:0,format:"name"};var n=u.rgb.exec(e);return n?{r:n[1],g:n[2],b:n[3]}:(n=u.rgba.exec(e))?{r:n[1],g:n[2],b:n[3],a:n[4]}:(n=u.hsl.exec(e))?{h:n[1],s:n[2],l:n[3]}:(n=u.hsla.exec(e))?{h:n[1],s:n[2],l:n[3],a:n[4]}:(n=u.hsv.exec(e))?{h:n[1],s:n[2],v:n[3]}:(n=u.hsva.exec(e))?{h:n[1],s:n[2],v:n[3],a:n[4]}:(n=u.hex8.exec(e))?{r:(0,r.VD)(n[1]),g:(0,r.VD)(n[2]),b:(0,r.VD)(n[3]),a:(0,r.T6)(n[4]),format:t?"name":"hex8"}:(n=u.hex6.exec(e))?{r:(0,r.VD)(n[1]),g:(0,r.VD)(n[2]),b:(0,r.VD)(n[3]),format:t?"name":"hex"}:(n=u.hex4.exec(e))?{r:(0,r.VD)(n[1]+n[1]),g:(0,r.VD)(n[2]+n[2]),b:(0,r.VD)(n[3]+n[3]),a:(0,r.T6)(n[4]+n[4]),format:t?"name":"hex8"}:!!(n=u.hex3.exec(e))&&{r:(0,r.VD)(n[1]+n[1]),g:(0,r.VD)(n[2]+n[2]),b:(0,r.VD)(n[3]+n[3]),format:t?"name":"hex"}}(e)),"object"==typeof e&&(d(e.r)&&d(e.g)&&d(e.b)?(t=(0,r.rW)(e.r,e.g,e.b),c=!0,p="%"===String(e.r).substr(-1)?"prgb":"rgb"):d(e.h)&&d(e.s)&&d(e.v)?(i=(0,a.JX)(e.s),l=(0,a.JX)(e.v),t=(0,r.WE)(e.h,i,l),c=!0,p="hsv"):d(e.h)&&d(e.s)&&d(e.l)&&(i=(0,a.JX)(e.s),s=(0,a.JX)(e.l),t=(0,r.ve)(e.h,i,s),c=!0,p="hsl"),Object.prototype.hasOwnProperty.call(e,"a")&&(n=e.a)),n=(0,a.Yq)(n),{ok:c,format:e.format||p,r:Math.min(255,Math.max(t.r,0)),g:Math.min(255,Math.max(t.g,0)),b:Math.min(255,Math.max(t.b,0)),a:n}}var l="(?:".concat("[-\\+]?\\d*\\.\\d+%?",")|(?:").concat("[-\\+]?\\d+%?",")"),s="[\\s|\\(]+(".concat(l,")[,|\\s]+(").concat(l,")[,|\\s]+(").concat(l,")\\s*\\)?"),c="[\\s|\\(]+(".concat(l,")[,|\\s]+(").concat(l,")[,|\\s]+(").concat(l,")[,|\\s]+(").concat(l,")\\s*\\)?"),u={CSS_UNIT:new RegExp(l),rgb:RegExp("rgb"+s),rgba:RegExp("rgba"+c),hsl:RegExp("hsl"+s),hsla:RegExp("hsla"+c),hsv:RegExp("hsv"+s),hsva:RegExp("hsva"+c),hex3:/^#?([0-9a-fA-F]{1})([0-9a-fA-F]{1})([0-9a-fA-F]{1})$/,hex6:/^#?([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})$/,hex4:/^#?([0-9a-fA-F]{1})([0-9a-fA-F]{1})([0-9a-fA-F]{1})([0-9a-fA-F]{1})$/,hex8:/^#?([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})$/};function d(e){return!!u.CSS_UNIT.exec(String(e))}},6336:function(e,t,n){n.d(t,{C:function(){return l}});var r=n(41785),o=n(6564),a=n(76991),i=n(27974),l=function(){function e(t,n){if(void 0===t&&(t=""),void 0===n&&(n={}),t instanceof e)return t;"number"==typeof t&&(t=(0,r.Yt)(t)),this.originalInput=t;var o,i=(0,a.uA)(t);this.originalInput=t,this.r=i.r,this.g=i.g,this.b=i.b,this.a=i.a,this.roundA=Math.round(100*this.a)/100,this.format=null!==(o=n.format)&&void 0!==o?o:i.format,this.gradientType=n.gradientType,this.r<1&&(this.r=Math.round(this.r)),this.g<1&&(this.g=Math.round(this.g)),this.b<1&&(this.b=Math.round(this.b)),this.isValid=i.ok}return e.prototype.isDark=function(){return 128>this.getBrightness()},e.prototype.isLight=function(){return!this.isDark()},e.prototype.getBrightness=function(){var e=this.toRgb();return(299*e.r+587*e.g+114*e.b)/1e3},e.prototype.getLuminance=function(){var e=this.toRgb(),t=e.r/255,n=e.g/255,r=e.b/255;return .2126*(t<=.03928?t/12.92:Math.pow((t+.055)/1.055,2.4))+.7152*(n<=.03928?n/12.92:Math.pow((n+.055)/1.055,2.4))+.0722*(r<=.03928?r/12.92:Math.pow((r+.055)/1.055,2.4))},e.prototype.getAlpha=function(){return this.a},e.prototype.setAlpha=function(e){return this.a=(0,i.Yq)(e),this.roundA=Math.round(100*this.a)/100,this},e.prototype.isMonochrome=function(){return 0===this.toHsl().s},e.prototype.toHsv=function(){var e=(0,r.py)(this.r,this.g,this.b);return{h:360*e.h,s:e.s,v:e.v,a:this.a}},e.prototype.toHsvString=function(){var e=(0,r.py)(this.r,this.g,this.b),t=Math.round(360*e.h),n=Math.round(100*e.s),o=Math.round(100*e.v);return 1===this.a?"hsv(".concat(t,", ").concat(n,"%, ").concat(o,"%)"):"hsva(".concat(t,", ").concat(n,"%, ").concat(o,"%, ").concat(this.roundA,")")},e.prototype.toHsl=function(){var e=(0,r.lC)(this.r,this.g,this.b);return{h:360*e.h,s:e.s,l:e.l,a:this.a}},e.prototype.toHslString=function(){var e=(0,r.lC)(this.r,this.g,this.b),t=Math.round(360*e.h),n=Math.round(100*e.s),o=Math.round(100*e.l);return 1===this.a?"hsl(".concat(t,", ").concat(n,"%, ").concat(o,"%)"):"hsla(".concat(t,", ").concat(n,"%, ").concat(o,"%, ").concat(this.roundA,")")},e.prototype.toHex=function(e){return void 0===e&&(e=!1),(0,r.vq)(this.r,this.g,this.b,e)},e.prototype.toHexString=function(e){return void 0===e&&(e=!1),"#"+this.toHex(e)},e.prototype.toHex8=function(e){return void 0===e&&(e=!1),(0,r.s)(this.r,this.g,this.b,this.a,e)},e.prototype.toHex8String=function(e){return void 0===e&&(e=!1),"#"+this.toHex8(e)},e.prototype.toHexShortString=function(e){return void 0===e&&(e=!1),1===this.a?this.toHexString(e):this.toHex8String(e)},e.prototype.toRgb=function(){return{r:Math.round(this.r),g:Math.round(this.g),b:Math.round(this.b),a:this.a}},e.prototype.toRgbString=function(){var e=Math.round(this.r),t=Math.round(this.g),n=Math.round(this.b);return 1===this.a?"rgb(".concat(e,", ").concat(t,", ").concat(n,")"):"rgba(".concat(e,", ").concat(t,", ").concat(n,", ").concat(this.roundA,")")},e.prototype.toPercentageRgb=function(){var e=function(e){return"".concat(Math.round(100*(0,i.sh)(e,255)),"%")};return{r:e(this.r),g:e(this.g),b:e(this.b),a:this.a}},e.prototype.toPercentageRgbString=function(){var e=function(e){return Math.round(100*(0,i.sh)(e,255))};return 1===this.a?"rgb(".concat(e(this.r),"%, ").concat(e(this.g),"%, ").concat(e(this.b),"%)"):"rgba(".concat(e(this.r),"%, ").concat(e(this.g),"%, ").concat(e(this.b),"%, ").concat(this.roundA,")")},e.prototype.toName=function(){if(0===this.a)return"transparent";if(this.a<1)return!1;for(var e="#"+(0,r.vq)(this.r,this.g,this.b,!1),t=0,n=Object.entries(o.R);t=0;return!t&&r&&(e.startsWith("hex")||"name"===e)?"name"===e&&0===this.a?this.toName():this.toRgbString():("rgb"===e&&(n=this.toRgbString()),"prgb"===e&&(n=this.toPercentageRgbString()),("hex"===e||"hex6"===e)&&(n=this.toHexString()),"hex3"===e&&(n=this.toHexString(!0)),"hex4"===e&&(n=this.toHex8String(!0)),"hex8"===e&&(n=this.toHex8String()),"name"===e&&(n=this.toName()),"hsl"===e&&(n=this.toHslString()),"hsv"===e&&(n=this.toHsvString()),n||this.toHexString())},e.prototype.toNumber=function(){return(Math.round(this.r)<<16)+(Math.round(this.g)<<8)+Math.round(this.b)},e.prototype.clone=function(){return new e(this.toString())},e.prototype.lighten=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.l+=t/100,n.l=(0,i.V2)(n.l),new e(n)},e.prototype.brighten=function(t){void 0===t&&(t=10);var n=this.toRgb();return n.r=Math.max(0,Math.min(255,n.r-Math.round(-(t/100*255)))),n.g=Math.max(0,Math.min(255,n.g-Math.round(-(t/100*255)))),n.b=Math.max(0,Math.min(255,n.b-Math.round(-(t/100*255)))),new e(n)},e.prototype.darken=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.l-=t/100,n.l=(0,i.V2)(n.l),new e(n)},e.prototype.tint=function(e){return void 0===e&&(e=10),this.mix("white",e)},e.prototype.shade=function(e){return void 0===e&&(e=10),this.mix("black",e)},e.prototype.desaturate=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.s-=t/100,n.s=(0,i.V2)(n.s),new e(n)},e.prototype.saturate=function(t){void 0===t&&(t=10);var n=this.toHsl();return n.s+=t/100,n.s=(0,i.V2)(n.s),new e(n)},e.prototype.greyscale=function(){return this.desaturate(100)},e.prototype.spin=function(t){var n=this.toHsl(),r=(n.h+t)%360;return n.h=r<0?360+r:r,new e(n)},e.prototype.mix=function(t,n){void 0===n&&(n=50);var r=this.toRgb(),o=new e(t).toRgb(),a=n/100;return new e({r:(o.r-r.r)*a+r.r,g:(o.g-r.g)*a+r.g,b:(o.b-r.b)*a+r.b,a:(o.a-r.a)*a+r.a})},e.prototype.analogous=function(t,n){void 0===t&&(t=6),void 0===n&&(n=30);var r=this.toHsl(),o=360/n,a=[this];for(r.h=(r.h-(o*t>>1)+720)%360;--t;)r.h=(r.h+o)%360,a.push(new e(r));return a},e.prototype.complement=function(){var t=this.toHsl();return t.h=(t.h+180)%360,new e(t)},e.prototype.monochromatic=function(t){void 0===t&&(t=6);for(var n=this.toHsv(),r=n.h,o=n.s,a=n.v,i=[],l=1/t;t--;)i.push(new e({h:r,s:o,v:a})),a=(a+l)%1;return i},e.prototype.splitcomplement=function(){var t=this.toHsl(),n=t.h;return[this,new e({h:(n+72)%360,s:t.s,l:t.l}),new e({h:(n+216)%360,s:t.s,l:t.l})]},e.prototype.onBackground=function(t){var n=this.toRgb(),r=new e(t).toRgb(),o=n.a+r.a*(1-n.a);return new e({r:(n.r*n.a+r.r*r.a*(1-n.a))/o,g:(n.g*n.a+r.g*r.a*(1-n.a))/o,b:(n.b*n.a+r.b*r.a*(1-n.a))/o,a:o})},e.prototype.triad=function(){return this.polyad(3)},e.prototype.tetrad=function(){return this.polyad(4)},e.prototype.polyad=function(t){for(var n=this.toHsl(),r=n.h,o=[this],a=360/t,i=1;iMath.abs(e-t))?1:e=360===t?(e<0?e%t+t:e%t)/parseFloat(String(t)):e%t/parseFloat(String(t))}function o(e){return Math.min(1,Math.max(0,e))}function a(e){return(isNaN(e=parseFloat(e))||e<0||e>1)&&(e=1),e}function i(e){return e<=1?"".concat(100*Number(e),"%"):e}function l(e){return 1===e.length?"0"+e:String(e)}n.d(t,{FZ:function(){return l},JX:function(){return i},V2:function(){return o},Yq:function(){return a},sh:function(){return r}})},88804:function(e,t,n){n.d(t,{Z:function(){return y}});var r,o=n(80406),a=n(64090),i=n(89542),l=n(22127);n(53850);var s=n(74084),c=a.createContext(null),u=n(63787),d=n(24800),p=[],f=n(24050);function m(e){var t=e.match(/^(.*)px$/),n=Number(null==t?void 0:t[1]);return Number.isNaN(n)?function(e){if("undefined"==typeof document)return 0;if(void 0===r){var t=document.createElement("div");t.style.width="100%",t.style.height="200px";var n=document.createElement("div"),o=n.style;o.position="absolute",o.top="0",o.left="0",o.pointerEvents="none",o.visibility="hidden",o.width="200px",o.height="150px",o.overflow="hidden",n.appendChild(t),document.body.appendChild(n);var a=t.offsetWidth;n.style.overflow="scroll";var i=t.offsetWidth;a===i&&(i=n.clientWidth),document.body.removeChild(n),r=a-i}return r}():n}var g="rc-util-locker-".concat(Date.now()),h=0,b=!1,v=function(e){return!1!==e&&((0,l.Z)()&&e?"string"==typeof e?document.querySelector(e):"function"==typeof e?e():e:null)},y=a.forwardRef(function(e,t){var n,r,y,E,w=e.open,S=e.autoLock,x=e.getContainer,O=(e.debug,e.autoDestroy),k=void 0===O||O,C=e.children,T=a.useState(w),A=(0,o.Z)(T,2),N=A[0],I=A[1],R=N||w;a.useEffect(function(){(k||w)&&I(w)},[w,k]);var _=a.useState(function(){return v(x)}),P=(0,o.Z)(_,2),M=P[0],L=P[1];a.useEffect(function(){var e=v(x);L(null!=e?e:null)});var D=function(e,t){var n=a.useState(function(){return(0,l.Z)()?document.createElement("div"):null}),r=(0,o.Z)(n,1)[0],i=a.useRef(!1),s=a.useContext(c),f=a.useState(p),m=(0,o.Z)(f,2),g=m[0],h=m[1],b=s||(i.current?void 0:function(e){h(function(t){return[e].concat((0,u.Z)(t))})});function v(){r.parentElement||document.body.appendChild(r),i.current=!0}function y(){var e;null===(e=r.parentElement)||void 0===e||e.removeChild(r),i.current=!1}return(0,d.Z)(function(){return e?s?s(v):v():y(),y},[e]),(0,d.Z)(function(){g.length&&(g.forEach(function(e){return e()}),h(p))},[g]),[r,b]}(R&&!M,0),j=(0,o.Z)(D,2),F=j[0],B=j[1],Z=null!=M?M:F;n=!!(S&&w&&(0,l.Z)()&&(Z===F||Z===document.body)),r=a.useState(function(){return h+=1,"".concat(g,"_").concat(h)}),y=(0,o.Z)(r,1)[0],(0,d.Z)(function(){if(n){var e=function(e){if("undefined"==typeof document||!e||!(e instanceof Element))return{width:0,height:0};var t=getComputedStyle(e,"::-webkit-scrollbar"),n=t.width,r=t.height;return{width:m(n),height:m(r)}}(document.body).width,t=document.body.scrollHeight>(window.innerHeight||document.documentElement.clientHeight)&&window.innerWidth>document.body.offsetWidth;(0,f.hq)("\nhtml body {\n overflow-y: hidden;\n ".concat(t?"width: calc(100% - ".concat(e,"px);"):"","\n}"),y)}else(0,f.jL)(y);return function(){(0,f.jL)(y)}},[n,y]);var U=null;C&&(0,s.Yr)(C)&&t&&(U=C.ref);var z=(0,s.x1)(U,t);if(!R||!(0,l.Z)()||void 0===M)return null;var H=!1===Z||("boolean"==typeof E&&(b=E),b),G=C;return t&&(G=a.cloneElement(C,{ref:z})),a.createElement(c.Provider,{value:B},H?G:(0,i.createPortal)(G,Z))})},44101:function(e,t,n){n.d(t,{Z:function(){return z}});var r=n(5239),o=n(80406),a=n(60635),i=n(88804),l=n(16480),s=n.n(l),c=n(46505),u=n(97472),d=n(74687),p=n(54811),f=n(91010),m=n(24800),g=n(76158),h=n(64090),b=n(14749),v=n(49367),y=n(74084);function E(e){var t=e.prefixCls,n=e.align,r=e.arrow,o=e.arrowPos,a=r||{},i=a.className,l=a.content,c=o.x,u=o.y,d=h.useRef();if(!n||!n.points)return null;var p={position:"absolute"};if(!1!==n.autoArrow){var f=n.points[0],m=n.points[1],g=f[0],b=f[1],v=m[0],y=m[1];g!==v&&["t","b"].includes(g)?"t"===g?p.top=0:p.bottom=0:p.top=void 0===u?0:u,b!==y&&["l","r"].includes(b)?"l"===b?p.left=0:p.right=0:p.left=void 0===c?0:c}return h.createElement("div",{ref:d,className:s()("".concat(t,"-arrow"),i),style:p},l)}function w(e){var t=e.prefixCls,n=e.open,r=e.zIndex,o=e.mask,a=e.motion;return o?h.createElement(v.ZP,(0,b.Z)({},a,{motionAppear:!0,visible:n,removeOnLeave:!0}),function(e){var n=e.className;return h.createElement("div",{style:{zIndex:r},className:s()("".concat(t,"-mask"),n)})}):null}var S=h.memo(function(e){return e.children},function(e,t){return t.cache}),x=h.forwardRef(function(e,t){var n=e.popup,a=e.className,i=e.prefixCls,l=e.style,u=e.target,d=e.onVisibleChanged,p=e.open,f=e.keepDom,g=e.fresh,x=e.onClick,O=e.mask,k=e.arrow,C=e.arrowPos,T=e.align,A=e.motion,N=e.maskMotion,I=e.forceRender,R=e.getPopupContainer,_=e.autoDestroy,P=e.portal,M=e.zIndex,L=e.onMouseEnter,D=e.onMouseLeave,j=e.onPointerEnter,F=e.ready,B=e.offsetX,Z=e.offsetY,U=e.offsetR,z=e.offsetB,H=e.onAlign,G=e.onPrepare,W=e.stretch,$=e.targetWidth,V=e.targetHeight,q="function"==typeof n?n():n,Y=p||f,K=(null==R?void 0:R.length)>0,X=h.useState(!R||!K),Q=(0,o.Z)(X,2),J=Q[0],ee=Q[1];if((0,m.Z)(function(){!J&&K&&u&&ee(!0)},[J,K,u]),!J)return null;var et="auto",en={left:"-1000vw",top:"-1000vh",right:et,bottom:et};if(F||!p){var er,eo=T.points,ea=T.dynamicInset||(null===(er=T._experimental)||void 0===er?void 0:er.dynamicInset),ei=ea&&"r"===eo[0][1],el=ea&&"b"===eo[0][0];ei?(en.right=U,en.left=et):(en.left=B,en.right=et),el?(en.bottom=z,en.top=et):(en.top=Z,en.bottom=et)}var es={};return W&&(W.includes("height")&&V?es.height=V:W.includes("minHeight")&&V&&(es.minHeight=V),W.includes("width")&&$?es.width=$:W.includes("minWidth")&&$&&(es.minWidth=$)),p||(es.pointerEvents="none"),h.createElement(P,{open:I||Y,getContainer:R&&function(){return R(u)},autoDestroy:_},h.createElement(w,{prefixCls:i,open:p,zIndex:M,mask:O,motion:N}),h.createElement(c.Z,{onResize:H,disabled:!p},function(e){return h.createElement(v.ZP,(0,b.Z)({motionAppear:!0,motionEnter:!0,motionLeave:!0,removeOnLeave:!1,forceRender:I,leavedClassName:"".concat(i,"-hidden")},A,{onAppearPrepare:G,onEnterPrepare:G,visible:p,onVisibleChanged:function(e){var t;null==A||null===(t=A.onVisibleChanged)||void 0===t||t.call(A,e),d(e)}}),function(n,o){var c=n.className,u=n.style,d=s()(i,c,a);return h.createElement("div",{ref:(0,y.sQ)(e,t,o),className:d,style:(0,r.Z)((0,r.Z)((0,r.Z)((0,r.Z)({"--arrow-x":"".concat(C.x||0,"px"),"--arrow-y":"".concat(C.y||0,"px")},en),es),u),{},{boxSizing:"border-box",zIndex:M},l),onMouseEnter:L,onMouseLeave:D,onPointerEnter:j,onClick:x},k&&h.createElement(E,{prefixCls:i,arrow:k,arrowPos:C,align:T}),h.createElement(S,{cache:!p&&!g},q))})}))}),O=h.forwardRef(function(e,t){var n=e.children,r=e.getTriggerDOMNode,o=(0,y.Yr)(n),a=h.useCallback(function(e){(0,y.mH)(t,r?r(e):e)},[r]),i=(0,y.x1)(a,n.ref);return o?h.cloneElement(n,{ref:i}):n}),k=h.createContext(null);function C(e){return e?Array.isArray(e)?e:[e]:[]}var T=n(73193);function A(e,t,n,r){return t||(n?{motionName:"".concat(e,"-").concat(n)}:r?{motionName:r}:null)}function N(e){return e.ownerDocument.defaultView}function I(e){for(var t=[],n=null==e?void 0:e.parentElement,r=["hidden","scroll","clip","auto"];n;){var o=N(n).getComputedStyle(n);[o.overflowX,o.overflowY,o.overflow].some(function(e){return r.includes(e)})&&t.push(n),n=n.parentElement}return t}function R(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:1;return Number.isNaN(e)?t:e}function _(e){return R(parseFloat(e),0)}function P(e,t){var n=(0,r.Z)({},e);return(t||[]).forEach(function(e){if(!(e instanceof HTMLBodyElement||e instanceof HTMLHtmlElement)){var t=N(e).getComputedStyle(e),r=t.overflow,o=t.overflowClipMargin,a=t.borderTopWidth,i=t.borderBottomWidth,l=t.borderLeftWidth,s=t.borderRightWidth,c=e.getBoundingClientRect(),u=e.offsetHeight,d=e.clientHeight,p=e.offsetWidth,f=e.clientWidth,m=_(a),g=_(i),h=_(l),b=_(s),v=R(Math.round(c.width/p*1e3)/1e3),y=R(Math.round(c.height/u*1e3)/1e3),E=m*y,w=h*v,S=0,x=0;if("clip"===r){var O=_(o);S=O*v,x=O*y}var k=c.x+w-S,C=c.y+E-x,T=k+c.width+2*S-w-b*v-(p-f-h-b)*v,A=C+c.height+2*x-E-g*y-(u-d-m-g)*y;n.left=Math.max(n.left,k),n.top=Math.max(n.top,C),n.right=Math.min(n.right,T),n.bottom=Math.min(n.bottom,A)}}),n}function M(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0,n="".concat(t),r=n.match(/^(.*)\%$/);return r?parseFloat(r[1])/100*e:parseFloat(n)}function L(e,t){var n=(0,o.Z)(t||[],2),r=n[0],a=n[1];return[M(e.width,r),M(e.height,a)]}function D(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"";return[e[0],e[1]]}function j(e,t){var n,r=t[0],o=t[1];return n="t"===r?e.y:"b"===r?e.y+e.height:e.y+e.height/2,{x:"l"===o?e.x:"r"===o?e.x+e.width:e.x+e.width/2,y:n}}function F(e,t){var n={t:"b",b:"t",l:"r",r:"l"};return e.map(function(e,r){return r===t?n[e]||"c":e}).join("")}var B=n(63787);n(53850);var Z=n(19223),U=["prefixCls","children","action","showAction","hideAction","popupVisible","defaultPopupVisible","onPopupVisibleChange","afterPopupVisibleChange","mouseEnterDelay","mouseLeaveDelay","focusDelay","blurDelay","mask","maskClosable","getPopupContainer","forceRender","autoDestroy","destroyPopupOnHide","popup","popupClassName","popupStyle","popupPlacement","builtinPlacements","popupAlign","zIndex","stretch","getPopupClassNameFromAlign","fresh","alignPoint","onPopupClick","onPopupAlign","arrow","popupMotion","maskMotion","popupTransitionName","popupAnimation","maskTransitionName","maskAnimation","className","getTriggerDOMNode"],z=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:i.Z;return h.forwardRef(function(t,n){var i,l,b,v,y,E,w,S,_,M,z,H,G,W,$,V,q,Y=t.prefixCls,K=void 0===Y?"rc-trigger-popup":Y,X=t.children,Q=t.action,J=t.showAction,ee=t.hideAction,et=t.popupVisible,en=t.defaultPopupVisible,er=t.onPopupVisibleChange,eo=t.afterPopupVisibleChange,ea=t.mouseEnterDelay,ei=t.mouseLeaveDelay,el=void 0===ei?.1:ei,es=t.focusDelay,ec=t.blurDelay,eu=t.mask,ed=t.maskClosable,ep=t.getPopupContainer,ef=t.forceRender,em=t.autoDestroy,eg=t.destroyPopupOnHide,eh=t.popup,eb=t.popupClassName,ev=t.popupStyle,ey=t.popupPlacement,eE=t.builtinPlacements,ew=void 0===eE?{}:eE,eS=t.popupAlign,ex=t.zIndex,eO=t.stretch,ek=t.getPopupClassNameFromAlign,eC=t.fresh,eT=t.alignPoint,eA=t.onPopupClick,eN=t.onPopupAlign,eI=t.arrow,eR=t.popupMotion,e_=t.maskMotion,eP=t.popupTransitionName,eM=t.popupAnimation,eL=t.maskTransitionName,eD=t.maskAnimation,ej=t.className,eF=t.getTriggerDOMNode,eB=(0,a.Z)(t,U),eZ=h.useState(!1),eU=(0,o.Z)(eZ,2),ez=eU[0],eH=eU[1];(0,m.Z)(function(){eH((0,g.Z)())},[]);var eG=h.useRef({}),eW=h.useContext(k),e$=h.useMemo(function(){return{registerSubPopup:function(e,t){eG.current[e]=t,null==eW||eW.registerSubPopup(e,t)}}},[eW]),eV=(0,f.Z)(),eq=h.useState(null),eY=(0,o.Z)(eq,2),eK=eY[0],eX=eY[1],eQ=(0,p.Z)(function(e){(0,u.S)(e)&&eK!==e&&eX(e),null==eW||eW.registerSubPopup(eV,e)}),eJ=h.useState(null),e0=(0,o.Z)(eJ,2),e1=e0[0],e2=e0[1],e4=h.useRef(null),e3=(0,p.Z)(function(e){(0,u.S)(e)&&e1!==e&&(e2(e),e4.current=e)}),e6=h.Children.only(X),e5=(null==e6?void 0:e6.props)||{},e8={},e9=(0,p.Z)(function(e){var t,n;return(null==e1?void 0:e1.contains(e))||(null===(t=(0,d.A)(e1))||void 0===t?void 0:t.host)===e||e===e1||(null==eK?void 0:eK.contains(e))||(null===(n=(0,d.A)(eK))||void 0===n?void 0:n.host)===e||e===eK||Object.values(eG.current).some(function(t){return(null==t?void 0:t.contains(e))||e===t})}),e7=A(K,eR,eM,eP),te=A(K,e_,eD,eL),tt=h.useState(en||!1),tn=(0,o.Z)(tt,2),tr=tn[0],to=tn[1],ta=null!=et?et:tr,ti=(0,p.Z)(function(e){void 0===et&&to(e)});(0,m.Z)(function(){to(et||!1)},[et]);var tl=h.useRef(ta);tl.current=ta;var ts=h.useRef([]);ts.current=[];var tc=(0,p.Z)(function(e){var t;ti(e),(null!==(t=ts.current[ts.current.length-1])&&void 0!==t?t:ta)!==e&&(ts.current.push(e),null==er||er(e))}),tu=h.useRef(),td=function(){clearTimeout(tu.current)},tp=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:0;td(),0===t?tc(e):tu.current=setTimeout(function(){tc(e)},1e3*t)};h.useEffect(function(){return td},[]);var tf=h.useState(!1),tm=(0,o.Z)(tf,2),tg=tm[0],th=tm[1];(0,m.Z)(function(e){(!e||ta)&&th(!0)},[ta]);var tb=h.useState(null),tv=(0,o.Z)(tb,2),ty=tv[0],tE=tv[1],tw=h.useState([0,0]),tS=(0,o.Z)(tw,2),tx=tS[0],tO=tS[1],tk=function(e){tO([e.clientX,e.clientY])},tC=(i=eT?tx:e1,l=h.useState({ready:!1,offsetX:0,offsetY:0,offsetR:0,offsetB:0,arrowX:0,arrowY:0,scaleX:1,scaleY:1,align:ew[ey]||{}}),v=(b=(0,o.Z)(l,2))[0],y=b[1],E=h.useRef(0),w=h.useMemo(function(){return eK?I(eK):[]},[eK]),S=h.useRef({}),ta||(S.current={}),_=(0,p.Z)(function(){if(eK&&i&&ta){var e,t,n,a,l,s,c,d=eK.ownerDocument,p=N(eK).getComputedStyle(eK),f=p.width,m=p.height,g=p.position,h=eK.style.left,b=eK.style.top,v=eK.style.right,E=eK.style.bottom,x=eK.style.overflow,O=(0,r.Z)((0,r.Z)({},ew[ey]),eS),k=d.createElement("div");if(null===(e=eK.parentElement)||void 0===e||e.appendChild(k),k.style.left="".concat(eK.offsetLeft,"px"),k.style.top="".concat(eK.offsetTop,"px"),k.style.position=g,k.style.height="".concat(eK.offsetHeight,"px"),k.style.width="".concat(eK.offsetWidth,"px"),eK.style.left="0",eK.style.top="0",eK.style.right="auto",eK.style.bottom="auto",eK.style.overflow="hidden",Array.isArray(i))n={x:i[0],y:i[1],width:0,height:0};else{var C=i.getBoundingClientRect();n={x:C.x,y:C.y,width:C.width,height:C.height}}var A=eK.getBoundingClientRect(),I=d.documentElement,_=I.clientWidth,M=I.clientHeight,B=I.scrollWidth,Z=I.scrollHeight,U=I.scrollTop,z=I.scrollLeft,H=A.height,G=A.width,W=n.height,$=n.width,V=O.htmlRegion,q="visible",Y="visibleFirst";"scroll"!==V&&V!==Y&&(V=q);var K=V===Y,X=P({left:-z,top:-U,right:B-z,bottom:Z-U},w),Q=P({left:0,top:0,right:_,bottom:M},w),J=V===q?Q:X,ee=K?Q:J;eK.style.left="auto",eK.style.top="auto",eK.style.right="0",eK.style.bottom="0";var et=eK.getBoundingClientRect();eK.style.left=h,eK.style.top=b,eK.style.right=v,eK.style.bottom=E,eK.style.overflow=x,null===(t=eK.parentElement)||void 0===t||t.removeChild(k);var en=R(Math.round(G/parseFloat(f)*1e3)/1e3),er=R(Math.round(H/parseFloat(m)*1e3)/1e3);if(!(0===en||0===er||(0,u.S)(i)&&!(0,T.Z)(i))){var eo=O.offset,ea=O.targetOffset,ei=L(A,eo),el=(0,o.Z)(ei,2),es=el[0],ec=el[1],eu=L(n,ea),ed=(0,o.Z)(eu,2),ep=ed[0],ef=ed[1];n.x-=ep,n.y-=ef;var em=O.points||[],eg=(0,o.Z)(em,2),eh=eg[0],eb=D(eg[1]),ev=D(eh),eE=j(n,eb),ex=j(A,ev),eO=(0,r.Z)({},O),ek=eE.x-ex.x+es,eC=eE.y-ex.y+ec,eT=tt(ek,eC),eA=tt(ek,eC,Q),eI=j(n,["t","l"]),eR=j(A,["t","l"]),e_=j(n,["b","r"]),eP=j(A,["b","r"]),eM=O.overflow||{},eL=eM.adjustX,eD=eM.adjustY,ej=eM.shiftX,eF=eM.shiftY,eB=function(e){return"boolean"==typeof e?e:e>=0};tn();var eZ=eB(eD),eU=ev[0]===eb[0];if(eZ&&"t"===ev[0]&&(l>ee.bottom||S.current.bt)){var ez=eC;eU?ez-=H-W:ez=eI.y-eP.y-ec;var eH=tt(ek,ez),eG=tt(ek,ez,Q);eH>eT||eH===eT&&(!K||eG>=eA)?(S.current.bt=!0,eC=ez,ec=-ec,eO.points=[F(ev,0),F(eb,0)]):S.current.bt=!1}if(eZ&&"b"===ev[0]&&(aeT||e$===eT&&(!K||eV>=eA)?(S.current.tb=!0,eC=eW,ec=-ec,eO.points=[F(ev,0),F(eb,0)]):S.current.tb=!1}var eq=eB(eL),eY=ev[1]===eb[1];if(eq&&"l"===ev[1]&&(c>ee.right||S.current.rl)){var eX=ek;eY?eX-=G-$:eX=eI.x-eP.x-es;var eQ=tt(eX,eC),eJ=tt(eX,eC,Q);eQ>eT||eQ===eT&&(!K||eJ>=eA)?(S.current.rl=!0,ek=eX,es=-es,eO.points=[F(ev,1),F(eb,1)]):S.current.rl=!1}if(eq&&"r"===ev[1]&&(seT||e1===eT&&(!K||e2>=eA)?(S.current.lr=!0,ek=e0,es=-es,eO.points=[F(ev,1),F(eb,1)]):S.current.lr=!1}tn();var e4=!0===ej?0:ej;"number"==typeof e4&&(sQ.right&&(ek-=c-Q.right-es,n.x>Q.right-e4&&(ek+=n.x-Q.right+e4)));var e3=!0===eF?0:eF;"number"==typeof e3&&(aQ.bottom&&(eC-=l-Q.bottom-ec,n.y>Q.bottom-e3&&(eC+=n.y-Q.bottom+e3)));var e6=A.x+ek,e5=A.y+eC,e8=n.x,e9=n.y;null==eN||eN(eK,eO);var e7=et.right-A.x-(ek+A.width),te=et.bottom-A.y-(eC+A.height);y({ready:!0,offsetX:ek/en,offsetY:eC/er,offsetR:e7/en,offsetB:te/er,arrowX:((Math.max(e6,e8)+Math.min(e6+G,e8+$))/2-e6)/en,arrowY:((Math.max(e5,e9)+Math.min(e5+H,e9+W))/2-e5)/er,scaleX:en,scaleY:er,align:eO})}function tt(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:J,r=A.x+e,o=A.y+t,a=Math.max(r,n.left),i=Math.max(o,n.top);return Math.max(0,(Math.min(r+G,n.right)-a)*(Math.min(o+H,n.bottom)-i))}function tn(){l=(a=A.y+eC)+H,c=(s=A.x+ek)+G}}}),M=function(){y(function(e){return(0,r.Z)((0,r.Z)({},e),{},{ready:!1})})},(0,m.Z)(M,[ey]),(0,m.Z)(function(){ta||M()},[ta]),[v.ready,v.offsetX,v.offsetY,v.offsetR,v.offsetB,v.arrowX,v.arrowY,v.scaleX,v.scaleY,v.align,function(){E.current+=1;var e=E.current;Promise.resolve().then(function(){E.current===e&&_()})}]),tT=(0,o.Z)(tC,11),tA=tT[0],tN=tT[1],tI=tT[2],tR=tT[3],t_=tT[4],tP=tT[5],tM=tT[6],tL=tT[7],tD=tT[8],tj=tT[9],tF=tT[10],tB=(z=void 0===Q?"hover":Q,h.useMemo(function(){var e=C(null!=J?J:z),t=C(null!=ee?ee:z),n=new Set(e),r=new Set(t);return ez&&(n.has("hover")&&(n.delete("hover"),n.add("click")),r.has("hover")&&(r.delete("hover"),r.add("click"))),[n,r]},[ez,z,J,ee])),tZ=(0,o.Z)(tB,2),tU=tZ[0],tz=tZ[1],tH=tU.has("click"),tG=tz.has("click")||tz.has("contextMenu"),tW=(0,p.Z)(function(){tg||tF()});H=function(){tl.current&&eT&&tG&&tp(!1)},(0,m.Z)(function(){if(ta&&e1&&eK){var e=I(e1),t=I(eK),n=N(eK),r=new Set([n].concat((0,B.Z)(e),(0,B.Z)(t)));function o(){tW(),H()}return r.forEach(function(e){e.addEventListener("scroll",o,{passive:!0})}),n.addEventListener("resize",o,{passive:!0}),tW(),function(){r.forEach(function(e){e.removeEventListener("scroll",o),n.removeEventListener("resize",o)})}}},[ta,e1,eK]),(0,m.Z)(function(){tW()},[tx,ey]),(0,m.Z)(function(){ta&&!(null!=ew&&ew[ey])&&tW()},[JSON.stringify(eS)]);var t$=h.useMemo(function(){var e=function(e,t,n,r){for(var o=n.points,a=Object.keys(e),i=0;i0&&void 0!==arguments[0]?arguments[0]:[],t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],n=arguments.length>2?arguments[2]:void 0;return n?e[0]===t[0]:e[0]===t[0]&&e[1]===t[1]}(null===(l=e[s])||void 0===l?void 0:l.points,o,r))return"".concat(t,"-placement-").concat(s)}return""}(ew,K,tj,eT);return s()(e,null==ek?void 0:ek(tj))},[tj,ek,ew,K,eT]);h.useImperativeHandle(n,function(){return{nativeElement:e4.current,forceAlign:tW}});var tV=h.useState(0),tq=(0,o.Z)(tV,2),tY=tq[0],tK=tq[1],tX=h.useState(0),tQ=(0,o.Z)(tX,2),tJ=tQ[0],t0=tQ[1],t1=function(){if(eO&&e1){var e=e1.getBoundingClientRect();tK(e.width),t0(e.height)}};function t2(e,t,n,r){e8[e]=function(o){var a;null==r||r(o),tp(t,n);for(var i=arguments.length,l=Array(i>1?i-1:0),s=1;s1?n-1:0),o=1;o1?n-1:0),o=1;o{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:"2.5"}),o.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M19 9l-7 7-7-7"}))}},8903:function(e,t,n){n.d(t,{Z:function(){return a}});var r=n(69703),o=n(64090);let a=e=>{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor"}),o.createElement("path",{fillRule:"evenodd",d:"M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z",clipRule:"evenodd"}))}},57750:function(e,t,n){n.d(t,{Z:function(){return eg}});var r=n(69703),o=n(64090),a=n(26587),i=n(65558),l=n(75504),s=n(30638),c=n(80509),u=n.n(c),d=n(5037),p=n.n(d),f=n(71292),m=n.n(f),g=n(96240),h=n.n(g),b=n(93574),v=n.n(b),y=n(72996),E=n(84487),w=n(7986),S=n(71594),x=n(68139),O=n(20757),k=n(9586),C=n(765),T=["layout","type","stroke","connectNulls","isRange","ref"];function A(e){return(A="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e})(e)}function N(){return(N=Object.assign?Object.assign.bind():function(e){for(var t=1;t=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0)&&Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}(a,T));return o.createElement(w.m,{clipPath:n?"url(#clipPath-".concat(r,")"):null},o.createElement(y.H,N({},(0,C.L6)(d,!0),{points:e,connectNulls:c,type:l,baseLine:t,layout:i,stroke:"none",className:"recharts-area-area"})),"none"!==s&&o.createElement(y.H,N({},(0,C.L6)(this.props,!1),{className:"recharts-area-curve",layout:i,type:l,connectNulls:c,fill:"none",points:e})),"none"!==s&&u&&o.createElement(y.H,N({},(0,C.L6)(this.props,!1),{className:"recharts-area-curve",layout:i,type:l,connectNulls:c,fill:"none",points:t})))}},{key:"renderAreaWithAnimation",value:function(e,t){var n=this,r=this.props,a=r.points,i=r.baseLine,l=r.isAnimationActive,c=r.animationBegin,u=r.animationDuration,d=r.animationEasing,p=r.animationId,f=this.state,g=f.prevPoints,b=f.prevBaseLine;return o.createElement(s.ZP,{begin:c,duration:u,isActive:l,easing:d,from:{t:0},to:{t:1},key:"area-".concat(p),onAnimationEnd:this.handleAnimationEnd,onAnimationStart:this.handleAnimationStart},function(r){var l=r.t;if(g){var s,c=g.length/a.length,u=a.map(function(e,t){var n=Math.floor(t*c);if(g[n]){var r=g[n],o=(0,O.k4)(r.x,e.x),a=(0,O.k4)(r.y,e.y);return R(R({},e),{},{x:o(l),y:a(l)})}return e});return s=(0,O.hj)(i)&&"number"==typeof i?(0,O.k4)(b,i)(l):m()(i)||h()(i)?(0,O.k4)(b,0)(l):i.map(function(e,t){var n=Math.floor(t*c);if(b[n]){var r=b[n],o=(0,O.k4)(r.x,e.x),a=(0,O.k4)(r.y,e.y);return R(R({},e),{},{x:o(l),y:a(l)})}return e}),n.renderAreaStatically(u,s,e,t)}return o.createElement(w.m,null,o.createElement("defs",null,o.createElement("clipPath",{id:"animationClipPath-".concat(t)},n.renderClipRect(l))),o.createElement(w.m,{clipPath:"url(#animationClipPath-".concat(t,")")},n.renderAreaStatically(a,i,e,t)))})}},{key:"renderArea",value:function(e,t){var n=this.props,r=n.points,o=n.baseLine,a=n.isAnimationActive,i=this.state,l=i.prevPoints,s=i.prevBaseLine,c=i.totalLength;return a&&r&&r.length&&(!l&&c>0||!v()(l,r)||!v()(s,o))?this.renderAreaWithAnimation(e,t):this.renderAreaStatically(r,o,e,t)}},{key:"render",value:function(){var e,t=this.props,n=t.hide,r=t.dot,a=t.points,i=t.className,s=t.top,c=t.left,u=t.xAxis,d=t.yAxis,p=t.width,f=t.height,g=t.isAnimationActive,h=t.id;if(n||!a||!a.length)return null;var b=this.state.isAnimationFinished,v=1===a.length,y=(0,l.Z)("recharts-area",i),E=u&&u.allowDataOverflow,x=d&&d.allowDataOverflow,O=E||x,k=m()(h)?this.id:h,T=null!==(e=(0,C.L6)(r,!1))&&void 0!==e?e:{r:3,strokeWidth:2},A=T.r,N=T.strokeWidth,I=((0,C.$k)(r)?r:{}).clipDot,R=void 0===I||I,_=2*(void 0===A?3:A)+(void 0===N?2:N);return o.createElement(w.m,{className:y},E||x?o.createElement("defs",null,o.createElement("clipPath",{id:"clipPath-".concat(k)},o.createElement("rect",{x:E?c:c-p/2,y:x?s:s-f/2,width:E?p:2*p,height:x?f:2*f})),!R&&o.createElement("clipPath",{id:"clipPath-dots-".concat(k)},o.createElement("rect",{x:c-_/2,y:s-_/2,width:p+_,height:f+_}))):null,v?null:this.renderArea(O,k),(r||v)&&this.renderDots(O,R,k),(!g||b)&&S.e.renderCallByParent(this.props,a))}}],r=[{key:"getDerivedStateFromProps",value:function(e,t){return e.animationId!==t.prevAnimationId?{prevAnimationId:e.animationId,curPoints:e.points,curBaseLine:e.baseLine,prevPoints:t.curPoints,prevBaseLine:t.curBaseLine}:e.points!==t.curPoints||e.baseLine!==t.curBaseLine?{curPoints:e.points,curBaseLine:e.baseLine}:null}}],n&&_(i.prototype,n),r&&_(i,r),Object.defineProperty(i,"prototype",{writable:!1}),i}(o.PureComponent);D(F,"displayName","Area"),D(F,"defaultProps",{stroke:"#3182bd",fill:"#3182bd",fillOpacity:.6,xAxisId:0,yAxisId:0,legendType:"line",connectNulls:!1,points:[],dot:!1,activeDot:!0,hide:!1,isAnimationActive:!x.x.isSsr,animationBegin:0,animationDuration:1500,animationEasing:"ease"}),D(F,"getBaseValue",function(e,t,n,r){var o=e.layout,a=e.baseValue,i=t.props.baseValue,l=null!=i?i:a;if((0,O.hj)(l)&&"number"==typeof l)return l;var s="horizontal"===o?r:n,c=s.scale.domain();if("number"===s.type){var u=Math.max(c[0],c[1]),d=Math.min(c[0],c[1]);return"dataMin"===l?d:"dataMax"===l?u:u<0?u:Math.max(Math.min(c[0],c[1]),0)}return"dataMin"===l?c[0]:"dataMax"===l?c[1]:c[0]}),D(F,"getComposedData",function(e){var t,n=e.props,r=e.item,o=e.xAxis,a=e.yAxis,i=e.xAxisTicks,l=e.yAxisTicks,s=e.bandSize,c=e.dataKey,u=e.stackedData,d=e.dataStartIndex,p=e.displayedData,f=e.offset,m=n.layout,g=u&&u.length,h=F.getBaseValue(n,r,o,a),b="horizontal"===m,v=!1,y=p.map(function(e,t){g?n=u[d+t]:Array.isArray(n=(0,k.F$)(e,c))?v=!0:n=[h,n];var n,r=null==n[1]||g&&null==(0,k.F$)(e,c);return b?{x:(0,k.Hv)({axis:o,ticks:i,bandSize:s,entry:e,index:t}),y:r?null:a.scale(n[1]),value:n,payload:e}:{x:r?null:o.scale(n[1]),y:(0,k.Hv)({axis:a,ticks:l,bandSize:s,entry:e,index:t}),value:n,payload:e}});return t=g||v?y.map(function(e){var t=Array.isArray(e.value)?e.value[0]:null;return b?{x:e.x,y:null!=t&&null!=e.y?a.scale(t):null}:{x:null!=t?o.scale(t):null,y:e.y}}):b?a.scale(h):o.scale(h),R({points:y,baseLine:t,layout:m,isRange:v},f)}),D(F,"renderDotItem",function(e,t){return o.isValidElement(e)?o.cloneElement(e,t):u()(e)?e(t):o.createElement(E.o,N({},t,{className:"recharts-area-dot"}))});var B=n(23356),Z=n(22983),U=n(12627),z=(0,i.z)({chartName:"AreaChart",GraphicalChild:F,axisComponents:[{axisType:"xAxis",AxisComp:B.K},{axisType:"yAxis",AxisComp:Z.B}],formatAxisMap:U.t9}),H=n(38333),G=n(10166),W=n(94866),$=n(99355),V=["type","layout","connectNulls","ref"];function q(e){return(q="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e})(e)}function Y(){return(Y=Object.assign?Object.assign.bind():function(e){for(var t=1;te.length)&&(t=e.length);for(var n=0,r=Array(t);na){s=[].concat(Q(r.slice(0,c)),[a-u]);break}var d=s.length%2==0?[0,l]:[l];return[].concat(Q(i.repeat(r,Math.floor(t/o))),Q(s),d).map(function(e){return"".concat(e,"px")}).join(", ")}),eo(en(e),"id",(0,O.EL)("recharts-line-")),eo(en(e),"pathRef",function(t){e.mainCurve=t}),eo(en(e),"handleAnimationEnd",function(){e.setState({isAnimationFinished:!0}),e.props.onAnimationEnd&&e.props.onAnimationEnd()}),eo(en(e),"handleAnimationStart",function(){e.setState({isAnimationFinished:!1}),e.props.onAnimationStart&&e.props.onAnimationStart()}),e}return n=[{key:"componentDidMount",value:function(){if(this.props.isAnimationActive){var e=this.getTotalLength();this.setState({totalLength:e})}}},{key:"componentDidUpdate",value:function(){if(this.props.isAnimationActive){var e=this.getTotalLength();e!==this.state.totalLength&&this.setState({totalLength:e})}}},{key:"getTotalLength",value:function(){var e=this.mainCurve;try{return e&&e.getTotalLength&&e.getTotalLength()||0}catch(e){return 0}}},{key:"renderErrorBar",value:function(e,t){if(this.props.isAnimationActive&&!this.state.isAnimationFinished)return null;var n=this.props,r=n.points,a=n.xAxis,i=n.yAxis,l=n.layout,s=n.children,c=(0,C.NN)(s,$.W);if(!c)return null;var u=function(e,t){return{x:e.x,y:e.y,value:e.value,errorVal:(0,k.F$)(e.payload,t)}};return o.createElement(w.m,{clipPath:e?"url(#clipPath-".concat(t,")"):null},c.map(function(e){return o.cloneElement(e,{key:"bar-".concat(e.props.dataKey),data:r,xAxis:a,yAxis:i,layout:l,dataPointFormatter:u})}))}},{key:"renderDots",value:function(e,t,n){if(this.props.isAnimationActive&&!this.state.isAnimationFinished)return null;var r=this.props,a=r.dot,l=r.points,s=r.dataKey,c=(0,C.L6)(this.props,!1),u=(0,C.L6)(a,!0),d=l.map(function(e,t){var n=X(X(X({key:"dot-".concat(t),r:3},c),u),{},{value:e.value,dataKey:s,cx:e.x,cy:e.y,index:t,payload:e.payload});return i.renderDotItem(a,n)}),p={clipPath:e?"url(#clipPath-".concat(t?"":"dots-").concat(n,")"):null};return o.createElement(w.m,Y({className:"recharts-line-dots",key:"dots"},p),d)}},{key:"renderCurveStatically",value:function(e,t,n,r){var a=this.props,i=a.type,l=a.layout,s=a.connectNulls,c=(a.ref,function(e,t){if(null==e)return{};var n,r,o=function(e,t){if(null==e)return{};var n,r,o={},a=Object.keys(e);for(r=0;r=0||(o[n]=e[n]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(r=0;r=0)&&Object.prototype.propertyIsEnumerable.call(e,n)&&(o[n]=e[n])}return o}(a,V)),u=X(X(X({},(0,C.L6)(c,!0)),{},{fill:"none",className:"recharts-line-curve",clipPath:t?"url(#clipPath-".concat(n,")"):null,points:e},r),{},{type:i,layout:l,connectNulls:s});return o.createElement(y.H,Y({},u,{pathRef:this.pathRef}))}},{key:"renderCurveWithAnimation",value:function(e,t){var n=this,r=this.props,a=r.points,i=r.strokeDasharray,l=r.isAnimationActive,c=r.animationBegin,u=r.animationDuration,d=r.animationEasing,p=r.animationId,f=r.animateNewValues,m=r.width,g=r.height,h=this.state,b=h.prevPoints,v=h.totalLength;return o.createElement(s.ZP,{begin:c,duration:u,isActive:l,easing:d,from:{t:0},to:{t:1},key:"line-".concat(p),onAnimationEnd:this.handleAnimationEnd,onAnimationStart:this.handleAnimationStart},function(r){var o,l=r.t;if(b){var s=b.length/a.length,c=a.map(function(e,t){var n=Math.floor(t*s);if(b[n]){var r=b[n],o=(0,O.k4)(r.x,e.x),a=(0,O.k4)(r.y,e.y);return X(X({},e),{},{x:o(l),y:a(l)})}if(f){var i=(0,O.k4)(2*m,e.x),c=(0,O.k4)(g/2,e.y);return X(X({},e),{},{x:i(l),y:c(l)})}return X(X({},e),{},{x:e.x,y:e.y})});return n.renderCurveStatically(c,e,t)}var u=(0,O.k4)(0,v)(l);if(i){var d="".concat(i).split(/[,\s]+/gim).map(function(e){return parseFloat(e)});o=n.getStrokeDasharray(u,v,d)}else o=n.generateSimpleStrokeDasharray(v,u);return n.renderCurveStatically(a,e,t,{strokeDasharray:o})})}},{key:"renderCurve",value:function(e,t){var n=this.props,r=n.points,o=n.isAnimationActive,a=this.state,i=a.prevPoints,l=a.totalLength;return o&&r&&r.length&&(!i&&l>0||!v()(i,r))?this.renderCurveWithAnimation(e,t):this.renderCurveStatically(r,e,t)}},{key:"render",value:function(){var e,t=this.props,n=t.hide,r=t.dot,a=t.points,i=t.className,s=t.xAxis,c=t.yAxis,u=t.top,d=t.left,p=t.width,f=t.height,g=t.isAnimationActive,h=t.id;if(n||!a||!a.length)return null;var b=this.state.isAnimationFinished,v=1===a.length,y=(0,l.Z)("recharts-line",i),E=s&&s.allowDataOverflow,x=c&&c.allowDataOverflow,O=E||x,k=m()(h)?this.id:h,T=null!==(e=(0,C.L6)(r,!1))&&void 0!==e?e:{r:3,strokeWidth:2},A=T.r,N=T.strokeWidth,I=((0,C.$k)(r)?r:{}).clipDot,R=void 0===I||I,_=2*(void 0===A?3:A)+(void 0===N?2:N);return o.createElement(w.m,{className:y},E||x?o.createElement("defs",null,o.createElement("clipPath",{id:"clipPath-".concat(k)},o.createElement("rect",{x:E?d:d-p/2,y:x?u:u-f/2,width:E?p:2*p,height:x?f:2*f})),!R&&o.createElement("clipPath",{id:"clipPath-dots-".concat(k)},o.createElement("rect",{x:d-_/2,y:u-_/2,width:p+_,height:f+_}))):null,!v&&this.renderCurve(O,k),this.renderErrorBar(O,k),(v||r)&&this.renderDots(O,R,k),(!g||b)&&S.e.renderCallByParent(this.props,a))}}],r=[{key:"getDerivedStateFromProps",value:function(e,t){return e.animationId!==t.prevAnimationId?{prevAnimationId:e.animationId,curPoints:e.points,prevPoints:t.curPoints}:e.points!==t.curPoints?{curPoints:e.points}:null}},{key:"repeat",value:function(e,t){for(var n=e.length%2!=0?[].concat(Q(e),[0]):e,r=[],o=0;o{let{data:n=[],categories:i=[],index:l,stack:s=!1,colors:c=ep.s,valueFormatter:u=em.Cj,startEndOnly:d=!1,showXAxis:p=!0,showYAxis:f=!0,yAxisWidth:m=56,intervalType:g="equidistantPreserveStart",showAnimation:h=!1,animationDuration:b=900,showTooltip:v=!0,showLegend:y=!0,showGridLines:w=!0,showGradient:S=!0,autoMinValue:x=!1,curveType:O="linear",minValue:k,maxValue:C,connectNulls:T=!1,allowDecimals:A=!0,noDataText:N,className:I,onValueChange:R,enableLegendSlider:_=!1,customTooltip:P,rotateLabelX:M,tickGap:L=5}=e,D=(0,r._T)(e,["data","categories","index","stack","colors","valueFormatter","startEndOnly","showXAxis","showYAxis","yAxisWidth","intervalType","showAnimation","animationDuration","showTooltip","showLegend","showGridLines","showGradient","autoMinValue","curveType","minValue","maxValue","connectNulls","allowDecimals","noDataText","className","onValueChange","enableLegendSlider","customTooltip","rotateLabelX","tickGap"]),j=(p||f)&&(!d||f)?20:0,[U,$]=(0,o.useState)(60),[V,q]=(0,o.useState)(void 0),[Y,K]=(0,o.useState)(void 0),X=(0,eu.me)(i,c),Q=(0,eu.i4)(x,k,C),J=!!R;function ee(e){J&&(e===Y&&!V||(0,eu.FB)(n,e)&&V&&V.dataKey===e?(K(void 0),null==R||R(null)):(K(e),null==R||R({eventType:"category",categoryClicked:e})),q(void 0))}return o.createElement("div",Object.assign({ref:t,className:(0,ef.q)("w-full h-80",I)},D),o.createElement(a.h,{className:"h-full w-full"},(null==n?void 0:n.length)?o.createElement(z,{data:n,onClick:J&&(Y||V)?()=>{q(void 0),K(void 0),null==R||R(null)}:void 0},w?o.createElement(H.q,{className:(0,ef.q)("stroke-1","stroke-tremor-border","dark:stroke-dark-tremor-border"),horizontal:!0,vertical:!1}):null,o.createElement(B.K,{padding:{left:j,right:j},hide:!p,dataKey:l,tick:{transform:"translate(0, 6)"},ticks:d?[n[0][l],n[n.length-1][l]]:void 0,fill:"",stroke:"",className:(0,ef.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),interval:d?"preserveStartEnd":g,tickLine:!1,axisLine:!1,minTickGap:L,angle:null==M?void 0:M.angle,dy:null==M?void 0:M.verticalShift,height:null==M?void 0:M.xAxisHeight}),o.createElement(Z.B,{width:m,hide:!f,axisLine:!1,tickLine:!1,type:"number",domain:Q,tick:{transform:"translate(-3, 0)"},fill:"",stroke:"",className:(0,ef.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickFormatter:u,allowDecimals:A}),o.createElement(G.u,{wrapperStyle:{outline:"none"},isAnimationActive:!1,cursor:{stroke:"#d1d5db",strokeWidth:1},content:v?e=>{let{active:t,payload:n,label:r}=e;return P?o.createElement(P,{payload:null==n?void 0:n.map(e=>{var t;return Object.assign(Object.assign({},e),{color:null!==(t=X.get(e.dataKey))&&void 0!==t?t:ed.fr.Gray})}),active:t,label:r}):o.createElement(es.ZP,{active:t,payload:n,label:r,valueFormatter:u,categoryColors:X})}:o.createElement(o.Fragment,null),position:{y:0}}),y?o.createElement(W.D,{verticalAlign:"top",height:U,content:e=>{let{payload:t}=e;return(0,el.Z)({payload:t},X,$,Y,J?e=>ee(e):void 0,_)}}):null,i.map(e=>{var t,n;return o.createElement("defs",{key:e},S?o.createElement("linearGradient",{className:(0,em.bM)(null!==(t=X.get(e))&&void 0!==t?t:ed.fr.Gray,ep.K.text).textColor,id:X.get(e),x1:"0",y1:"0",x2:"0",y2:"1"},o.createElement("stop",{offset:"5%",stopColor:"currentColor",stopOpacity:V||Y&&Y!==e?.15:.4}),o.createElement("stop",{offset:"95%",stopColor:"currentColor",stopOpacity:0})):o.createElement("linearGradient",{className:(0,em.bM)(null!==(n=X.get(e))&&void 0!==n?n:ed.fr.Gray,ep.K.text).textColor,id:X.get(e),x1:"0",y1:"0",x2:"0",y2:"1"},o.createElement("stop",{stopColor:"currentColor",stopOpacity:V||Y&&Y!==e?.1:.3})))}),i.map(e=>{var t;return o.createElement(F,{className:(0,em.bM)(null!==(t=X.get(e))&&void 0!==t?t:ed.fr.Gray,ep.K.text).strokeColor,strokeOpacity:V||Y&&Y!==e?.3:1,activeDot:e=>{var t;let{cx:r,cy:a,stroke:i,strokeLinecap:l,strokeLinejoin:s,strokeWidth:c,dataKey:u}=e;return o.createElement(E.o,{className:(0,ef.q)("stroke-tremor-background dark:stroke-dark-tremor-background",R?"cursor-pointer":"",(0,em.bM)(null!==(t=X.get(u))&&void 0!==t?t:ed.fr.Gray,ep.K.text).fillColor),cx:r,cy:a,r:5,fill:"",stroke:i,strokeLinecap:l,strokeLinejoin:s,strokeWidth:c,onClick:(t,r)=>{r.stopPropagation(),J&&(e.index===(null==V?void 0:V.index)&&e.dataKey===(null==V?void 0:V.dataKey)||(0,eu.FB)(n,e.dataKey)&&Y&&Y===e.dataKey?(K(void 0),q(void 0),null==R||R(null)):(K(e.dataKey),q({index:e.index,dataKey:e.dataKey}),null==R||R(Object.assign({eventType:"dot",categoryClicked:e.dataKey},e.payload))))}})},dot:t=>{var r;let{stroke:a,strokeLinecap:i,strokeLinejoin:l,strokeWidth:s,cx:c,cy:u,dataKey:d,index:p}=t;return(0,eu.FB)(n,e)&&!(V||Y&&Y!==e)||(null==V?void 0:V.index)===p&&(null==V?void 0:V.dataKey)===e?o.createElement(E.o,{key:p,cx:c,cy:u,r:5,stroke:a,fill:"",strokeLinecap:i,strokeLinejoin:l,strokeWidth:s,className:(0,ef.q)("stroke-tremor-background dark:stroke-dark-tremor-background",R?"cursor-pointer":"",(0,em.bM)(null!==(r=X.get(d))&&void 0!==r?r:ed.fr.Gray,ep.K.text).fillColor)}):o.createElement(o.Fragment,{key:p})},key:e,name:e,type:O,dataKey:e,stroke:"",fill:"url(#".concat(X.get(e),")"),strokeWidth:2,strokeLinejoin:"round",strokeLinecap:"round",isAnimationActive:h,animationDuration:b,stackId:s?"a":void 0,connectNulls:T})}),R?i.map(e=>o.createElement(ei,{className:(0,ef.q)("cursor-pointer"),strokeOpacity:0,key:e,name:e,type:O,dataKey:e,stroke:"transparent",fill:"transparent",legendType:"none",tooltipType:"none",strokeWidth:12,connectNulls:T,onClick:(e,t)=>{t.stopPropagation();let{name:n}=e;ee(n)}})):null):o.createElement(ec.Z,{noDataText:N})))});eg.displayName="AreaChart"},44041:function(e,t,n){n.d(t,{Z:function(){return x}});var r=n(69703),o=n(54942),a=n(2898),i=n(99250),l=n(65492),s=n(64090),c=n(26587),u=n(65558),d=n(28485),p=n(23356),f=n(22983),m=n(12627),g=(0,u.z)({chartName:"BarChart",GraphicalChild:d.$,defaultTooltipEventType:"axis",validateTooltipEventTypes:["axis","item"],axisComponents:[{axisType:"xAxis",AxisComp:p.K},{axisType:"yAxis",AxisComp:f.B}],formatAxisMap:m.t9}),h=n(38333),b=n(10166),v=n(94866),y=n(17280),E=n(30470),w=n(77448),S=n(36342);let x=s.forwardRef((e,t)=>{let{data:n=[],categories:u=[],index:m,colors:x=a.s,valueFormatter:O=l.Cj,layout:k="horizontal",stack:C=!1,relative:T=!1,startEndOnly:A=!1,animationDuration:N=900,showAnimation:I=!1,showXAxis:R=!0,showYAxis:_=!0,yAxisWidth:P=56,intervalType:M="equidistantPreserveStart",showTooltip:L=!0,showLegend:D=!0,showGridLines:j=!0,autoMinValue:F=!1,minValue:B,maxValue:Z,allowDecimals:U=!0,noDataText:z,onValueChange:H,enableLegendSlider:G=!1,customTooltip:W,rotateLabelX:$,tickGap:V=5,className:q}=e,Y=(0,r._T)(e,["data","categories","index","colors","valueFormatter","layout","stack","relative","startEndOnly","animationDuration","showAnimation","showXAxis","showYAxis","yAxisWidth","intervalType","showTooltip","showLegend","showGridLines","autoMinValue","minValue","maxValue","allowDecimals","noDataText","onValueChange","enableLegendSlider","customTooltip","rotateLabelX","tickGap","className"]),K=R||_?20:0,[X,Q]=(0,s.useState)(60),J=(0,S.me)(u,x),[ee,et]=s.useState(void 0),[en,er]=(0,s.useState)(void 0),eo=!!H;function ea(e,t,n){var r,o,a,i;n.stopPropagation(),H&&((0,S.vZ)(ee,Object.assign(Object.assign({},e.payload),{value:e.value}))?(er(void 0),et(void 0),null==H||H(null)):(er(null===(o=null===(r=e.tooltipPayload)||void 0===r?void 0:r[0])||void 0===o?void 0:o.dataKey),et(Object.assign(Object.assign({},e.payload),{value:e.value})),null==H||H(Object.assign({eventType:"bar",categoryClicked:null===(i=null===(a=e.tooltipPayload)||void 0===a?void 0:a[0])||void 0===i?void 0:i.dataKey},e.payload))))}let ei=(0,S.i4)(F,B,Z);return s.createElement("div",Object.assign({ref:t,className:(0,i.q)("w-full h-80",q)},Y),s.createElement(c.h,{className:"h-full w-full"},(null==n?void 0:n.length)?s.createElement(g,{data:n,stackOffset:C?"sign":T?"expand":"none",layout:"vertical"===k?"vertical":"horizontal",onClick:eo&&(en||ee)?()=>{et(void 0),er(void 0),null==H||H(null)}:void 0},j?s.createElement(h.q,{className:(0,i.q)("stroke-1","stroke-tremor-border","dark:stroke-dark-tremor-border"),horizontal:"vertical"!==k,vertical:"vertical"===k}):null,"vertical"!==k?s.createElement(p.K,{padding:{left:K,right:K},hide:!R,dataKey:m,interval:A?"preserveStartEnd":M,tick:{transform:"translate(0, 6)"},ticks:A?[n[0][m],n[n.length-1][m]]:void 0,fill:"",stroke:"",className:(0,i.q)("mt-4 text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickLine:!1,axisLine:!1,angle:null==$?void 0:$.angle,dy:null==$?void 0:$.verticalShift,height:null==$?void 0:$.xAxisHeight,minTickGap:V}):s.createElement(p.K,{hide:!R,type:"number",tick:{transform:"translate(-3, 0)"},domain:ei,fill:"",stroke:"",className:(0,i.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickLine:!1,axisLine:!1,tickFormatter:O,minTickGap:V,allowDecimals:U,angle:null==$?void 0:$.angle,dy:null==$?void 0:$.verticalShift,height:null==$?void 0:$.xAxisHeight}),"vertical"!==k?s.createElement(f.B,{width:P,hide:!_,axisLine:!1,tickLine:!1,type:"number",domain:ei,tick:{transform:"translate(-3, 0)"},fill:"",stroke:"",className:(0,i.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content"),tickFormatter:T?e=>"".concat((100*e).toString()," %"):O,allowDecimals:U}):s.createElement(f.B,{width:P,hide:!_,dataKey:m,axisLine:!1,tickLine:!1,ticks:A?[n[0][m],n[n.length-1][m]]:void 0,type:"category",interval:"preserveStartEnd",tick:{transform:"translate(0, 6)"},fill:"",stroke:"",className:(0,i.q)("text-tremor-label","fill-tremor-content","dark:fill-dark-tremor-content")}),s.createElement(b.u,{wrapperStyle:{outline:"none"},isAnimationActive:!1,cursor:{fill:"#d1d5db",opacity:"0.15"},content:L?e=>{let{active:t,payload:n,label:r}=e;return W?s.createElement(W,{payload:null==n?void 0:n.map(e=>{var t;return Object.assign(Object.assign({},e),{color:null!==(t=J.get(e.dataKey))&&void 0!==t?t:o.fr.Gray})}),active:t,label:r}):s.createElement(E.ZP,{active:t,payload:n,label:r,valueFormatter:O,categoryColors:J})}:s.createElement(s.Fragment,null),position:{y:0}}),D?s.createElement(v.D,{verticalAlign:"top",height:X,content:e=>{let{payload:t}=e;return(0,y.Z)({payload:t},J,Q,en,eo?e=>{eo&&(e!==en||ee?(er(e),null==H||H({eventType:"category",categoryClicked:e})):(er(void 0),null==H||H(null)),et(void 0))}:void 0,G)}}):null,u.map(e=>{var t;return s.createElement(d.$,{className:(0,i.q)((0,l.bM)(null!==(t=J.get(e))&&void 0!==t?t:o.fr.Gray,a.K.background).fillColor,H?"cursor-pointer":""),key:e,name:e,type:"linear",stackId:C||T?"a":void 0,dataKey:e,fill:"",isAnimationActive:I,animationDuration:N,shape:e=>((e,t,n,r)=>{let{fillOpacity:o,name:a,payload:i,value:l}=e,{x:c,width:u,y:d,height:p}=e;return"horizontal"===r&&p<0?(d+=p,p=Math.abs(p)):"vertical"===r&&u<0&&(c+=u,u=Math.abs(u)),s.createElement("rect",{x:c,y:d,width:u,height:p,opacity:t||n&&n!==a?(0,S.vZ)(t,Object.assign(Object.assign({},i),{value:l}))?o:.3:o})})(e,ee,en,k),onClick:ea})})):s.createElement(w.Z,{noDataText:z})))});x.displayName="BarChart"},17280:function(e,t,n){n.d(t,{Z:function(){return g}});var r=n(64090);let o=(e,t)=>{let[n,o]=(0,r.useState)(t);(0,r.useEffect)(()=>{let t=()=>{o(window.innerWidth),e()};return t(),window.addEventListener("resize",t),()=>window.removeEventListener("resize",t)},[e,n])};var a=n(69703),i=n(2898),l=n(99250),s=n(65492);let c=e=>{var t=(0,a._T)(e,[]);return r.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"}),r.createElement("path",{d:"M8 12L14 6V18L8 12Z"}))},u=e=>{var t=(0,a._T)(e,[]);return r.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"}),r.createElement("path",{d:"M16 12L10 18V6L16 12Z"}))},d=(0,s.fn)("Legend"),p=e=>{let{name:t,color:n,onClick:o,activeLegend:a}=e,c=!!o;return r.createElement("li",{className:(0,l.q)(d("legendItem"),"group inline-flex items-center px-2 py-0.5 rounded-tremor-small transition whitespace-nowrap",c?"cursor-pointer":"cursor-default","text-tremor-content",c?"hover:bg-tremor-background-subtle":"","dark:text-dark-tremor-content",c?"dark:hover:bg-dark-tremor-background-subtle":""),onClick:e=>{e.stopPropagation(),null==o||o(t,n)}},r.createElement("svg",{className:(0,l.q)("flex-none h-2 w-2 mr-1.5",(0,s.bM)(n,i.K.text).textColor,a&&a!==t?"opacity-40":"opacity-100"),fill:"currentColor",viewBox:"0 0 8 8"},r.createElement("circle",{cx:4,cy:4,r:4})),r.createElement("p",{className:(0,l.q)("whitespace-nowrap truncate text-tremor-default","text-tremor-content",c?"group-hover:text-tremor-content-emphasis":"","dark:text-dark-tremor-content",a&&a!==t?"opacity-40":"opacity-100",c?"dark:group-hover:text-dark-tremor-content-emphasis":"")},t))},f=e=>{let{icon:t,onClick:n,disabled:o}=e,[a,i]=r.useState(!1),s=r.useRef(null);return r.useEffect(()=>(a?s.current=setInterval(()=>{null==n||n()},300):clearInterval(s.current),()=>clearInterval(s.current)),[a,n]),(0,r.useEffect)(()=>{o&&(clearInterval(s.current),i(!1))},[o]),r.createElement("button",{type:"button",className:(0,l.q)(d("legendSliderButton"),"w-5 group inline-flex items-center truncate rounded-tremor-small transition",o?"cursor-not-allowed":"cursor-pointer",o?"text-tremor-content-subtle":"text-tremor-content hover:text-tremor-content-emphasis hover:bg-tremor-background-subtle",o?"dark:text-dark-tremor-subtle":"dark:text-dark-tremor dark:hover:text-tremor-content-emphasis dark:hover:bg-dark-tremor-background-subtle"),disabled:o,onClick:e=>{e.stopPropagation(),null==n||n()},onMouseDown:e=>{e.stopPropagation(),i(!0)},onMouseUp:e=>{e.stopPropagation(),i(!1)}},r.createElement(t,{className:"w-full"}))},m=r.forwardRef((e,t)=>{var n,o;let{categories:s,colors:m=i.s,className:g,onClickLegendItem:h,activeLegend:b,enableLegendSlider:v=!1}=e,y=(0,a._T)(e,["categories","colors","className","onClickLegendItem","activeLegend","enableLegendSlider"]),E=r.useRef(null),[w,S]=r.useState(null),[x,O]=r.useState(null),k=r.useRef(null),C=(0,r.useCallback)(()=>{let e=null==E?void 0:E.current;e&&S({left:e.scrollLeft>0,right:e.scrollWidth-e.clientWidth>e.scrollLeft})},[S]),T=(0,r.useCallback)(e=>{var t;let n=null==E?void 0:E.current,r=null!==(t=null==n?void 0:n.clientWidth)&&void 0!==t?t:0;n&&v&&(n.scrollTo({left:"left"===e?n.scrollLeft-r:n.scrollLeft+r,behavior:"smooth"}),setTimeout(()=>{C()},400))},[v,C]);r.useEffect(()=>{let e=e=>{"ArrowLeft"===e?T("left"):"ArrowRight"===e&&T("right")};return x?(e(x),k.current=setInterval(()=>{e(x)},300)):clearInterval(k.current),()=>clearInterval(k.current)},[x,T]);let A=e=>{e.stopPropagation(),"ArrowLeft"!==e.key&&"ArrowRight"!==e.key||(e.preventDefault(),O(e.key))},N=e=>{e.stopPropagation(),O(null)};return r.useEffect(()=>{let e=null==E?void 0:E.current;return v&&(C(),null==e||e.addEventListener("keydown",A),null==e||e.addEventListener("keyup",N)),()=>{null==e||e.removeEventListener("keydown",A),null==e||e.removeEventListener("keyup",N)}},[C,v]),r.createElement("ol",Object.assign({ref:t,className:(0,l.q)(d("root"),"relative overflow-hidden",g)},y),r.createElement("div",{ref:E,tabIndex:0,className:(0,l.q)("h-full flex",v?(null==w?void 0:w.right)||(null==w?void 0:w.left)?"pl-4 pr-12 items-center overflow-auto snap-mandatory [&::-webkit-scrollbar]:hidden [scrollbar-width:none]":"":"flex-wrap")},s.map((e,t)=>r.createElement(p,{key:"item-".concat(t),name:e,color:m[t],onClick:h,activeLegend:b}))),v&&((null==w?void 0:w.right)||(null==w?void 0:w.left))?r.createElement(r.Fragment,null,r.createElement("div",{className:(0,l.q)("absolute top-0 bottom-0 left-0 w-4 bg-gradient-to-r from-white to-transparent pointer-events-none")}),r.createElement("div",{className:(0,l.q)("absolute top-0 bottom-0 right-10 w-4 bg-gradient-to-r from-transparent to-white pointer-events-none")}),r.createElement("div",{className:(0,l.q)("absolute flex top-0 pr-1 bottom-0 right-0 items-center justify-center h-full bg-tremor-background")},r.createElement(f,{icon:c,onClick:()=>{O(null),T("left")},disabled:!(null==w?void 0:w.left)}),r.createElement(f,{icon:u,onClick:()=>{O(null),T("right")},disabled:!(null==w?void 0:w.right)}))):null)});m.displayName="Legend";let g=(e,t,n,a,i,l)=>{let{payload:s}=e,c=(0,r.useRef)(null);o(()=>{var e,t;n((t=null===(e=c.current)||void 0===e?void 0:e.clientHeight)?Number(t)+20:60)});let u=s.filter(e=>"none"!==e.type);return r.createElement("div",{ref:c,className:"flex items-center justify-end"},r.createElement(m,{categories:u.map(e=>e.value),colors:u.map(e=>t.get(e.value)),onClickLegendItem:i,activeLegend:a,enableLegendSlider:l}))}},30470:function(e,t,n){n.d(t,{ZP:function(){return u}});var r=n(64090),o=n(54942),a=n(2898),i=n(99250),l=n(65492);let s=e=>{let{children:t}=e;return r.createElement("div",{className:(0,i.q)("rounded-tremor-default text-tremor-default border","bg-tremor-background shadow-tremor-dropdown border-tremor-border","dark:bg-dark-tremor-background dark:shadow-dark-tremor-dropdown dark:border-dark-tremor-border")},t)},c=e=>{let{value:t,name:n,color:o}=e;return r.createElement("div",{className:"flex items-center justify-between space-x-8"},r.createElement("div",{className:"flex items-center space-x-2"},r.createElement("span",{className:(0,i.q)("shrink-0 rounded-tremor-full border-2 h-3 w-3","border-tremor-background shadow-tremor-card","dark:border-dark-tremor-background dark:shadow-dark-tremor-card",(0,l.bM)(o,a.K.background).bgColor)}),r.createElement("p",{className:(0,i.q)("text-right whitespace-nowrap","text-tremor-content","dark:text-dark-tremor-content")},n)),r.createElement("p",{className:(0,i.q)("font-medium tabular-nums text-right whitespace-nowrap","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis")},t))},u=e=>{let{active:t,payload:n,label:a,categoryColors:l,valueFormatter:u}=e;if(t&&n){let e=n.filter(e=>"none"!==e.type);return r.createElement(s,null,r.createElement("div",{className:(0,i.q)("border-tremor-border border-b px-4 py-2","dark:border-dark-tremor-border")},r.createElement("p",{className:(0,i.q)("font-medium","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis")},a)),r.createElement("div",{className:(0,i.q)("px-4 py-2 space-y-1")},e.map((e,t)=>{var n;let{value:a,name:i}=e;return r.createElement(c,{key:"id-".concat(t),value:u(a),name:i,color:null!==(n=l.get(i))&&void 0!==n?n:o.fr.Blue})})))}return null}},77448:function(e,t,n){n.d(t,{Z:function(){return p}});var r=n(99250),o=n(64090),a=n(69703);let i=(0,n(65492).fn)("Flex"),l={start:"justify-start",end:"justify-end",center:"justify-center",between:"justify-between",around:"justify-around",evenly:"justify-evenly"},s={start:"items-start",end:"items-end",center:"items-center",baseline:"items-baseline",stretch:"items-stretch"},c={row:"flex-row",col:"flex-col","row-reverse":"flex-row-reverse","col-reverse":"flex-col-reverse"},u=o.forwardRef((e,t)=>{let{flexDirection:n="row",justifyContent:u="between",alignItems:d="center",children:p,className:f}=e,m=(0,a._T)(e,["flexDirection","justifyContent","alignItems","children","className"]);return o.createElement("div",Object.assign({ref:t,className:(0,r.q)(i("root"),"flex w-full",c[n],l[u],s[d],f)},m),p)});u.displayName="Flex";var d=n(71801);let p=e=>{let{noDataText:t="No data"}=e;return o.createElement(u,{alignItems:"center",justifyContent:"center",className:(0,r.q)("w-full h-full border border-dashed rounded-tremor-default","border-tremor-border","dark:border-dark-tremor-border")},o.createElement(d.Z,{className:(0,r.q)("text-tremor-content","dark:text-dark-tremor-content")},t))}},36342:function(e,t,n){n.d(t,{FB:function(){return a},i4:function(){return o},me:function(){return r},vZ:function(){return function e(t,n){if(t===n)return!0;if("object"!=typeof t||"object"!=typeof n||null===t||null===n)return!1;let r=Object.keys(t),o=Object.keys(n);if(r.length!==o.length)return!1;for(let a of r)if(!o.includes(a)||!e(t[a],n[a]))return!1;return!0}}});let r=(e,t)=>{let n=new Map;return e.forEach((e,r)=>{n.set(e,t[r])}),n},o=(e,t,n)=>[e?"auto":null!=t?t:0,null!=n?n:"auto"];function a(e,t){let n=[];for(let r of e)if(Object.prototype.hasOwnProperty.call(r,t)&&(n.push(r[t]),n.length>1))return!1;return!0}},5:function(e,t,n){n.d(t,{Z:function(){return f}});var r=n(69703),o=n(64090),a=n(58437),i=n(54942),l=n(2898),s=n(99250),c=n(65492);let u={xs:{paddingX:"px-2",paddingY:"py-0.5",fontSize:"text-xs"},sm:{paddingX:"px-2.5",paddingY:"py-0.5",fontSize:"text-sm"},md:{paddingX:"px-3",paddingY:"py-0.5",fontSize:"text-md"},lg:{paddingX:"px-3.5",paddingY:"py-0.5",fontSize:"text-lg"},xl:{paddingX:"px-4",paddingY:"py-1",fontSize:"text-xl"}},d={xs:{height:"h-4",width:"w-4"},sm:{height:"h-4",width:"w-4"},md:{height:"h-4",width:"w-4"},lg:{height:"h-5",width:"w-5"},xl:{height:"h-6",width:"w-6"}},p=(0,c.fn)("Badge"),f=o.forwardRef((e,t)=>{let{color:n,icon:f,size:m=i.u8.SM,tooltip:g,className:h,children:b}=e,v=(0,r._T)(e,["color","icon","size","tooltip","className","children"]),y=f||null,{tooltipProps:E,getReferenceProps:w}=(0,a.l)();return o.createElement("span",Object.assign({ref:(0,c.lq)([t,E.refs.setReference]),className:(0,s.q)(p("root"),"w-max flex-shrink-0 inline-flex justify-center items-center cursor-default rounded-tremor-full",n?(0,s.q)((0,c.bM)(n,l.K.background).bgColor,(0,c.bM)(n,l.K.text).textColor,"bg-opacity-20 dark:bg-opacity-25"):(0,s.q)("bg-tremor-brand-muted text-tremor-brand-emphasis","dark:bg-dark-tremor-brand-muted dark:text-dark-tremor-brand-emphasis"),u[m].paddingX,u[m].paddingY,u[m].fontSize,h)},w,v),o.createElement(a.Z,Object.assign({text:g},E)),y?o.createElement(y,{className:(0,s.q)(p("icon"),"shrink-0 -ml-1 mr-1.5",d[m].height,d[m].width)}):null,o.createElement("p",{className:(0,s.q)(p("text"),"text-sm whitespace-nowrap")},b))});f.displayName="Badge"},61244:function(e,t,n){n.d(t,{Z:function(){return g}});var r=n(69703),o=n(64090),a=n(58437),i=n(54942),l=n(99250),s=n(65492),c=n(2898);let u={xs:{paddingX:"px-1.5",paddingY:"py-1.5"},sm:{paddingX:"px-1.5",paddingY:"py-1.5"},md:{paddingX:"px-2",paddingY:"py-2"},lg:{paddingX:"px-2",paddingY:"py-2"},xl:{paddingX:"px-2.5",paddingY:"py-2.5"}},d={xs:{height:"h-3",width:"w-3"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-7",width:"w-7"},xl:{height:"h-9",width:"w-9"}},p={simple:{rounded:"",border:"",ring:"",shadow:""},light:{rounded:"rounded-tremor-default",border:"",ring:"",shadow:""},shadow:{rounded:"rounded-tremor-default",border:"border",ring:"",shadow:"shadow-tremor-card dark:shadow-dark-tremor-card"},solid:{rounded:"rounded-tremor-default",border:"border-2",ring:"ring-1",shadow:""},outlined:{rounded:"rounded-tremor-default",border:"border",ring:"ring-2",shadow:""}},f=(e,t)=>{switch(e){case"simple":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:"",borderColor:"",ringColor:""};case"light":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-brand-muted dark:bg-dark-tremor-brand-muted",borderColor:"",ringColor:""};case"shadow":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:"border-tremor-border dark:border-dark-tremor-border",ringColor:""};case"solid":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-brand dark:bg-dark-tremor-brand",borderColor:"border-tremor-brand-inverted dark:border-dark-tremor-brand-inverted",ringColor:"ring-tremor-ring dark:ring-dark-tremor-ring"};case"outlined":return{textColor:t?(0,s.bM)(t,c.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",bgColor:t?(0,l.q)((0,s.bM)(t,c.K.background).bgColor,"bg-opacity-20"):"bg-tremor-background dark:bg-dark-tremor-background",borderColor:t?(0,s.bM)(t,c.K.ring).borderColor:"border-tremor-brand-subtle dark:border-dark-tremor-brand-subtle",ringColor:t?(0,l.q)((0,s.bM)(t,c.K.ring).ringColor,"ring-opacity-40"):"ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted"}}},m=(0,s.fn)("Icon"),g=o.forwardRef((e,t)=>{let{icon:n,variant:c="simple",tooltip:g,size:h=i.u8.SM,color:b,className:v}=e,y=(0,r._T)(e,["icon","variant","tooltip","size","color","className"]),E=f(c,b),{tooltipProps:w,getReferenceProps:S}=(0,a.l)();return o.createElement("span",Object.assign({ref:(0,s.lq)([t,w.refs.setReference]),className:(0,l.q)(m("root"),"inline-flex flex-shrink-0 items-center",E.bgColor,E.textColor,E.borderColor,E.ringColor,p[c].rounded,p[c].border,p[c].shadow,p[c].ring,u[h].paddingX,u[h].paddingY,v)},S,y),o.createElement(a.Z,Object.assign({text:g},w)),o.createElement(n,{className:(0,l.q)(m("icon"),"shrink-0",d[h].height,d[h].width)}))});g.displayName="Icon"},16450:function(e,t,n){n.d(t,{Z:function(){return A}});var r=n(69703),o=n(58437),a=n(64090),i=n(70444),l=n(89988),s=n(89542),c={disabled:!1},u=a.createContext(null),d="unmounted",p="exited",f="entering",m="entered",g="exiting",h=function(e){function t(t,n){r=e.call(this,t,n)||this;var r,o,a=n&&!n.isMounting?t.enter:t.appear;return r.appearStatus=null,t.in?a?(o=p,r.appearStatus=f):o=m:o=t.unmountOnExit||t.mountOnEnter?d:p,r.state={status:o},r.nextCallback=null,r}t.prototype=Object.create(e.prototype),t.prototype.constructor=t,(0,l.Z)(t,e),t.getDerivedStateFromProps=function(e,t){return e.in&&t.status===d?{status:p}:null};var n=t.prototype;return n.componentDidMount=function(){this.updateStatus(!0,this.appearStatus)},n.componentDidUpdate=function(e){var t=null;if(e!==this.props){var n=this.state.status;this.props.in?n!==f&&n!==m&&(t=f):(n===f||n===m)&&(t=g)}this.updateStatus(!1,t)},n.componentWillUnmount=function(){this.cancelNextCallback()},n.getTimeouts=function(){var e,t,n,r=this.props.timeout;return e=t=n=r,null!=r&&"number"!=typeof r&&(e=r.exit,t=r.enter,n=void 0!==r.appear?r.appear:t),{exit:e,enter:t,appear:n}},n.updateStatus=function(e,t){if(void 0===e&&(e=!1),null!==t){if(this.cancelNextCallback(),t===f){if(this.props.unmountOnExit||this.props.mountOnEnter){var n=this.props.nodeRef?this.props.nodeRef.current:s.findDOMNode(this);n&&n.scrollTop}this.performEnter(e)}else this.performExit()}else this.props.unmountOnExit&&this.state.status===p&&this.setState({status:d})},n.performEnter=function(e){var t=this,n=this.props.enter,r=this.context?this.context.isMounting:e,o=this.props.nodeRef?[r]:[s.findDOMNode(this),r],a=o[0],i=o[1],l=this.getTimeouts(),u=r?l.appear:l.enter;if(!e&&!n||c.disabled){this.safeSetState({status:m},function(){t.props.onEntered(a)});return}this.props.onEnter(a,i),this.safeSetState({status:f},function(){t.props.onEntering(a,i),t.onTransitionEnd(u,function(){t.safeSetState({status:m},function(){t.props.onEntered(a,i)})})})},n.performExit=function(){var e=this,t=this.props.exit,n=this.getTimeouts(),r=this.props.nodeRef?void 0:s.findDOMNode(this);if(!t||c.disabled){this.safeSetState({status:p},function(){e.props.onExited(r)});return}this.props.onExit(r),this.safeSetState({status:g},function(){e.props.onExiting(r),e.onTransitionEnd(n.exit,function(){e.safeSetState({status:p},function(){e.props.onExited(r)})})})},n.cancelNextCallback=function(){null!==this.nextCallback&&(this.nextCallback.cancel(),this.nextCallback=null)},n.safeSetState=function(e,t){t=this.setNextCallback(t),this.setState(e,t)},n.setNextCallback=function(e){var t=this,n=!0;return this.nextCallback=function(r){n&&(n=!1,t.nextCallback=null,e(r))},this.nextCallback.cancel=function(){n=!1},this.nextCallback},n.onTransitionEnd=function(e,t){this.setNextCallback(t);var n=this.props.nodeRef?this.props.nodeRef.current:s.findDOMNode(this),r=null==e&&!this.props.addEndListener;if(!n||r){setTimeout(this.nextCallback,0);return}if(this.props.addEndListener){var o=this.props.nodeRef?[this.nextCallback]:[n,this.nextCallback],a=o[0],i=o[1];this.props.addEndListener(a,i)}null!=e&&setTimeout(this.nextCallback,e)},n.render=function(){var e=this.state.status;if(e===d)return null;var t=this.props,n=t.children,r=(t.in,t.mountOnEnter,t.unmountOnExit,t.appear,t.enter,t.exit,t.timeout,t.addEndListener,t.onEnter,t.onEntering,t.onEntered,t.onExit,t.onExiting,t.onExited,t.nodeRef,(0,i.Z)(t,["children","in","mountOnEnter","unmountOnExit","appear","enter","exit","timeout","addEndListener","onEnter","onEntering","onEntered","onExit","onExiting","onExited","nodeRef"]));return a.createElement(u.Provider,{value:null},"function"==typeof n?n(e,r):a.cloneElement(a.Children.only(n),r))},t}(a.Component);function b(){}h.contextType=u,h.propTypes={},h.defaultProps={in:!1,mountOnEnter:!1,unmountOnExit:!1,appear:!1,enter:!0,exit:!0,onEnter:b,onEntering:b,onEntered:b,onExit:b,onExiting:b,onExited:b},h.UNMOUNTED=d,h.EXITED=p,h.ENTERING=f,h.ENTERED=m,h.EXITING=g;var v=n(54942),y=n(99250),E=n(65492);let w=e=>{var t=(0,r._T)(e,[]);return a.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"currentColor"}),a.createElement("path",{fill:"none",d:"M0 0h24v24H0z"}),a.createElement("path",{d:"M18.364 5.636L16.95 7.05A7 7 0 1 0 19 12h2a9 9 0 1 1-2.636-6.364z"}))};var S=n(2898);let x={xs:{height:"h-4",width:"w-4"},sm:{height:"h-5",width:"w-5"},md:{height:"h-5",width:"w-5"},lg:{height:"h-6",width:"w-6"},xl:{height:"h-6",width:"w-6"}},O=e=>"light"!==e?{xs:{paddingX:"px-2.5",paddingY:"py-1.5",fontSize:"text-xs"},sm:{paddingX:"px-4",paddingY:"py-2",fontSize:"text-sm"},md:{paddingX:"px-4",paddingY:"py-2",fontSize:"text-md"},lg:{paddingX:"px-4",paddingY:"py-2.5",fontSize:"text-lg"},xl:{paddingX:"px-4",paddingY:"py-3",fontSize:"text-xl"}}:{xs:{paddingX:"",paddingY:"",fontSize:"text-xs"},sm:{paddingX:"",paddingY:"",fontSize:"text-sm"},md:{paddingX:"",paddingY:"",fontSize:"text-md"},lg:{paddingX:"",paddingY:"",fontSize:"text-lg"},xl:{paddingX:"",paddingY:"",fontSize:"text-xl"}},k=(e,t)=>{switch(e){case"primary":return{textColor:t?(0,E.bM)("white").textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",hoverTextColor:t?(0,E.bM)("white").textColor:"text-tremor-brand-inverted dark:text-dark-tremor-brand-inverted",bgColor:t?(0,E.bM)(t,S.K.background).bgColor:"bg-tremor-brand dark:bg-dark-tremor-brand",hoverBgColor:t?(0,E.bM)(t,S.K.darkBackground).hoverBgColor:"hover:bg-tremor-brand-emphasis dark:hover:bg-dark-tremor-brand-emphasis",borderColor:t?(0,E.bM)(t,S.K.border).borderColor:"border-tremor-brand dark:border-dark-tremor-brand",hoverBorderColor:t?(0,E.bM)(t,S.K.darkBorder).hoverBorderColor:"hover:border-tremor-brand-emphasis dark:hover:border-dark-tremor-brand-emphasis"};case"secondary":return{textColor:t?(0,E.bM)(t,S.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",hoverTextColor:t?(0,E.bM)(t,S.K.text).textColor:"hover:text-tremor-brand-emphasis dark:hover:text-dark-tremor-brand-emphasis",bgColor:(0,E.bM)("transparent").bgColor,hoverBgColor:t?(0,y.q)((0,E.bM)(t,S.K.background).hoverBgColor,"hover:bg-opacity-20 dark:hover:bg-opacity-20"):"hover:bg-tremor-brand-faint dark:hover:bg-dark-tremor-brand-faint",borderColor:t?(0,E.bM)(t,S.K.border).borderColor:"border-tremor-brand dark:border-dark-tremor-brand"};case"light":return{textColor:t?(0,E.bM)(t,S.K.text).textColor:"text-tremor-brand dark:text-dark-tremor-brand",hoverTextColor:t?(0,E.bM)(t,S.K.darkText).hoverTextColor:"hover:text-tremor-brand-emphasis dark:hover:text-dark-tremor-brand-emphasis",bgColor:(0,E.bM)("transparent").bgColor,borderColor:"",hoverBorderColor:""}}},C=(0,E.fn)("Button"),T=e=>{let{loading:t,iconSize:n,iconPosition:r,Icon:o,needMargin:i,transitionState:l}=e,s=i?r===v.zS.Left?(0,y.q)("-ml-1","mr-1.5"):(0,y.q)("-mr-1","ml-1.5"):"",c=(0,y.q)("w-0 h-0"),u={default:c,entering:c,entered:n,exiting:n,exited:c};return t?a.createElement(w,{className:(0,y.q)(C("icon"),"animate-spin shrink-0",s,u.default,u[l]),style:{transition:"width 150ms"}}):a.createElement(o,{className:(0,y.q)(C("icon"),"shrink-0",n,s)})},A=a.forwardRef((e,t)=>{let{icon:n,iconPosition:i=v.zS.Left,size:l=v.u8.SM,color:s,variant:c="primary",disabled:u,loading:d=!1,loadingText:p,children:f,tooltip:m,className:g}=e,b=(0,r._T)(e,["icon","iconPosition","size","color","variant","disabled","loading","loadingText","children","tooltip","className"]),w=d||u,S=void 0!==n||d,A=d&&p,N=!(!f&&!A),I=(0,y.q)(x[l].height,x[l].width),R="light"!==c?(0,y.q)("rounded-tremor-default border","shadow-tremor-input","dark:shadow-dark-tremor-input"):"",_=k(c,s),P=O(c)[l],{tooltipProps:M,getReferenceProps:L}=(0,o.l)(300);return a.createElement(h,{in:d,timeout:50},e=>a.createElement("button",Object.assign({ref:(0,E.lq)([t,M.refs.setReference]),className:(0,y.q)(C("root"),"flex-shrink-0 inline-flex justify-center items-center group font-medium outline-none",R,P.paddingX,P.paddingY,P.fontSize,_.textColor,_.bgColor,_.borderColor,_.hoverBorderColor,w?"opacity-50 cursor-not-allowed":(0,y.q)(k(c,s).hoverTextColor,k(c,s).hoverBgColor,k(c,s).hoverBorderColor),g),disabled:w},L,b),a.createElement(o.Z,Object.assign({text:m},M)),S&&i!==v.zS.Right?a.createElement(T,{loading:d,iconSize:I,iconPosition:i,Icon:n,transitionState:e,needMargin:N}):null,A||f?a.createElement("span",{className:(0,y.q)(C("text"),"text-sm whitespace-nowrap")},A?p:f):null,S&&i===v.zS.Right?a.createElement(T,{loading:d,iconSize:I,iconPosition:i,Icon:n,transitionState:e,needMargin:N}):null))});A.displayName="Button"},5474:function(e,t,n){n.d(t,{Z:function(){return eD}});var r,o,a=n(69703),i=n(64090),l=n(73832),s=n(10641),c=n(15740),u=n(92381),d=n(39790),p=n(85235),f=n(71679),m=n(37130),g=n(71454),h=n(31820),b=n(36601),v=n(83839),y=n(37700),E=n(88358),w=n(84152),S=n(48803),x=n(72640),O=n(94819),k=n(18318),C=n(67409),T=((r=T||{})[r.Open=0]="Open",r[r.Closed=1]="Closed",r),A=((o=A||{})[o.TogglePopover=0]="TogglePopover",o[o.ClosePopover=1]="ClosePopover",o[o.SetButton=2]="SetButton",o[o.SetButtonId=3]="SetButtonId",o[o.SetPanel=4]="SetPanel",o[o.SetPanelId=5]="SetPanelId",o);let N={0:e=>{let t={...e,popoverState:(0,x.E)(e.popoverState,{0:1,1:0})};return 0===t.popoverState&&(t.__demoMode=!1),t},1:e=>1===e.popoverState?e:{...e,popoverState:1},2:(e,t)=>e.button===t.button?e:{...e,button:t.button},3:(e,t)=>e.buttonId===t.buttonId?e:{...e,buttonId:t.buttonId},4:(e,t)=>e.panel===t.panel?e:{...e,panel:t.panel},5:(e,t)=>e.panelId===t.panelId?e:{...e,panelId:t.panelId}},I=(0,i.createContext)(null);function R(e){let t=(0,i.useContext)(I);if(null===t){let t=Error("<".concat(e," /> is missing a parent component."));throw Error.captureStackTrace&&Error.captureStackTrace(t,R),t}return t}I.displayName="PopoverContext";let _=(0,i.createContext)(null);function P(e){let t=(0,i.useContext)(_);if(null===t){let t=Error("<".concat(e," /> is missing a parent component."));throw Error.captureStackTrace&&Error.captureStackTrace(t,P),t}return t}_.displayName="PopoverAPIContext";let M=(0,i.createContext)(null);function L(){return(0,i.useContext)(M)}M.displayName="PopoverGroupContext";let D=(0,i.createContext)(null);function j(e,t){return(0,x.E)(t.type,N,e,t)}D.displayName="PopoverPanelContext";let F=k.AN.RenderStrategy|k.AN.Static,B=k.AN.RenderStrategy|k.AN.Static,Z=Object.assign((0,k.yV)(function(e,t){var n;let{__demoMode:r=!1,...o}=e,a=(0,i.useRef)(null),u=(0,b.T)(t,(0,b.h)(e=>{a.current=e})),d=(0,i.useRef)([]),g=(0,i.useReducer)(j,{__demoMode:r,popoverState:r?0:1,buttons:d,button:null,buttonId:null,panel:null,panelId:null,beforePanelSentinel:(0,i.createRef)(),afterPanelSentinel:(0,i.createRef)()}),[{popoverState:v,button:y,buttonId:w,panel:O,panelId:C,beforePanelSentinel:T,afterPanelSentinel:A},N]=g,R=(0,m.i)(null!=(n=a.current)?n:y),P=(0,i.useMemo)(()=>{if(!y||!O)return!1;for(let e of document.querySelectorAll("body > *"))if(Number(null==e?void 0:e.contains(y))^Number(null==e?void 0:e.contains(O)))return!0;let e=(0,S.GO)(),t=e.indexOf(y),n=(t+e.length-1)%e.length,r=(t+1)%e.length,o=e[n],a=e[r];return!O.contains(o)&&!O.contains(a)},[y,O]),M=(0,p.E)(w),F=(0,p.E)(C),B=(0,i.useMemo)(()=>({buttonId:M,panelId:F,close:()=>N({type:1})}),[M,F,N]),Z=L(),U=null==Z?void 0:Z.registerPopover,z=(0,s.z)(()=>{var e;return null!=(e=null==Z?void 0:Z.isFocusWithinPopoverGroup())?e:(null==R?void 0:R.activeElement)&&((null==y?void 0:y.contains(R.activeElement))||(null==O?void 0:O.contains(R.activeElement)))});(0,i.useEffect)(()=>null==U?void 0:U(B),[U,B]);let[H,G]=(0,l.k)(),W=(0,h.v)({mainTreeNodeRef:null==Z?void 0:Z.mainTreeNodeRef,portals:H,defaultContainers:[y,O]});(0,c.O)(null==R?void 0:R.defaultView,"focus",e=>{var t,n,r,o;e.target!==window&&e.target instanceof HTMLElement&&0===v&&(z()||y&&O&&(W.contains(e.target)||null!=(n=null==(t=T.current)?void 0:t.contains)&&n.call(t,e.target)||null!=(o=null==(r=A.current)?void 0:r.contains)&&o.call(r,e.target)||N({type:1})))},!0),(0,f.O)(W.resolveContainers,(e,t)=>{N({type:1}),(0,S.sP)(t,S.tJ.Loose)||(e.preventDefault(),null==y||y.focus())},0===v);let $=(0,s.z)(e=>{N({type:1});let t=e?e instanceof HTMLElement?e:"current"in e&&e.current instanceof HTMLElement?e.current:y:y;null==t||t.focus()}),V=(0,i.useMemo)(()=>({close:$,isPortalled:P}),[$,P]),q=(0,i.useMemo)(()=>({open:0===v,close:$}),[v,$]);return i.createElement(D.Provider,{value:null},i.createElement(I.Provider,{value:g},i.createElement(_.Provider,{value:V},i.createElement(E.up,{value:(0,x.E)(v,{0:E.ZM.Open,1:E.ZM.Closed})},i.createElement(G,null,(0,k.sY)({ourProps:{ref:u},theirProps:o,slot:q,defaultTag:"div",name:"Popover"}),i.createElement(W.MainTreeNode,null))))))}),{Button:(0,k.yV)(function(e,t){let n=(0,u.M)(),{id:r="headlessui-popover-button-".concat(n),...o}=e,[a,l]=R("Popover.Button"),{isPortalled:c}=P("Popover.Button"),d=(0,i.useRef)(null),p="headlessui-focus-sentinel-".concat((0,u.M)()),f=L(),h=null==f?void 0:f.closeOthers,E=null!==(0,i.useContext)(D);(0,i.useEffect)(()=>{if(!E)return l({type:3,buttonId:r}),()=>{l({type:3,buttonId:null})}},[E,r,l]);let[O]=(0,i.useState)(()=>Symbol()),T=(0,b.T)(d,t,E?null:e=>{if(e)a.buttons.current.push(O);else{let e=a.buttons.current.indexOf(O);-1!==e&&a.buttons.current.splice(e,1)}a.buttons.current.length>1&&console.warn("You are already using a but only 1 is supported."),e&&l({type:2,button:e})}),A=(0,b.T)(d,t),N=(0,m.i)(d),I=(0,s.z)(e=>{var t,n,r;if(E){if(1===a.popoverState)return;switch(e.key){case C.R.Space:case C.R.Enter:e.preventDefault(),null==(n=(t=e.target).click)||n.call(t),l({type:1}),null==(r=a.button)||r.focus()}}else switch(e.key){case C.R.Space:case C.R.Enter:e.preventDefault(),e.stopPropagation(),1===a.popoverState&&(null==h||h(a.buttonId)),l({type:0});break;case C.R.Escape:if(0!==a.popoverState)return null==h?void 0:h(a.buttonId);if(!d.current||null!=N&&N.activeElement&&!d.current.contains(N.activeElement))return;e.preventDefault(),e.stopPropagation(),l({type:1})}}),_=(0,s.z)(e=>{E||e.key===C.R.Space&&e.preventDefault()}),M=(0,s.z)(t=>{var n,r;(0,w.P)(t.currentTarget)||e.disabled||(E?(l({type:1}),null==(n=a.button)||n.focus()):(t.preventDefault(),t.stopPropagation(),1===a.popoverState&&(null==h||h(a.buttonId)),l({type:0}),null==(r=a.button)||r.focus()))}),j=(0,s.z)(e=>{e.preventDefault(),e.stopPropagation()}),F=0===a.popoverState,B=(0,i.useMemo)(()=>({open:F}),[F]),Z=(0,g.f)(e,d),U=E?{ref:A,type:Z,onKeyDown:I,onClick:M}:{ref:T,id:a.buttonId,type:Z,"aria-expanded":0===a.popoverState,"aria-controls":a.panel?a.panelId:void 0,onKeyDown:I,onKeyUp:_,onClick:M,onMouseDown:j},z=(0,v.l)(),H=(0,s.z)(()=>{let e=a.panel;e&&(0,x.E)(z.current,{[v.N.Forwards]:()=>(0,S.jA)(e,S.TO.First),[v.N.Backwards]:()=>(0,S.jA)(e,S.TO.Last)})===S.fE.Error&&(0,S.jA)((0,S.GO)().filter(e=>"true"!==e.dataset.headlessuiFocusGuard),(0,x.E)(z.current,{[v.N.Forwards]:S.TO.Next,[v.N.Backwards]:S.TO.Previous}),{relativeTo:a.button})});return i.createElement(i.Fragment,null,(0,k.sY)({ourProps:U,theirProps:o,slot:B,defaultTag:"button",name:"Popover.Button"}),F&&!E&&c&&i.createElement(y._,{id:p,features:y.A.Focusable,"data-headlessui-focus-guard":!0,as:"button",type:"button",onFocus:H}))}),Overlay:(0,k.yV)(function(e,t){let n=(0,u.M)(),{id:r="headlessui-popover-overlay-".concat(n),...o}=e,[{popoverState:a},l]=R("Popover.Overlay"),c=(0,b.T)(t),d=(0,E.oJ)(),p=null!==d?(d&E.ZM.Open)===E.ZM.Open:0===a,f=(0,s.z)(e=>{if((0,w.P)(e.currentTarget))return e.preventDefault();l({type:1})}),m=(0,i.useMemo)(()=>({open:0===a}),[a]);return(0,k.sY)({ourProps:{ref:c,id:r,"aria-hidden":!0,onClick:f},theirProps:o,slot:m,defaultTag:"div",features:F,visible:p,name:"Popover.Overlay"})}),Panel:(0,k.yV)(function(e,t){let n=(0,u.M)(),{id:r="headlessui-popover-panel-".concat(n),focus:o=!1,...a}=e,[l,c]=R("Popover.Panel"),{close:p,isPortalled:f}=P("Popover.Panel"),g="headlessui-focus-sentinel-before-".concat((0,u.M)()),h="headlessui-focus-sentinel-after-".concat((0,u.M)()),w=(0,i.useRef)(null),O=(0,b.T)(w,t,e=>{c({type:4,panel:e})}),T=(0,m.i)(w),A=(0,k.Y2)();(0,d.e)(()=>(c({type:5,panelId:r}),()=>{c({type:5,panelId:null})}),[r,c]);let N=(0,E.oJ)(),I=null!==N?(N&E.ZM.Open)===E.ZM.Open:0===l.popoverState,_=(0,s.z)(e=>{var t;if(e.key===C.R.Escape){if(0!==l.popoverState||!w.current||null!=T&&T.activeElement&&!w.current.contains(T.activeElement))return;e.preventDefault(),e.stopPropagation(),c({type:1}),null==(t=l.button)||t.focus()}});(0,i.useEffect)(()=>{var t;e.static||1===l.popoverState&&(null==(t=e.unmount)||t)&&c({type:4,panel:null})},[l.popoverState,e.unmount,e.static,c]),(0,i.useEffect)(()=>{if(l.__demoMode||!o||0!==l.popoverState||!w.current)return;let e=null==T?void 0:T.activeElement;w.current.contains(e)||(0,S.jA)(w.current,S.TO.First)},[l.__demoMode,o,w,l.popoverState]);let M=(0,i.useMemo)(()=>({open:0===l.popoverState,close:p}),[l,p]),L={ref:O,id:r,onKeyDown:_,onBlur:o&&0===l.popoverState?e=>{var t,n,r,o,a;let i=e.relatedTarget;i&&w.current&&(null!=(t=w.current)&&t.contains(i)||(c({type:1}),(null!=(r=null==(n=l.beforePanelSentinel.current)?void 0:n.contains)&&r.call(n,i)||null!=(a=null==(o=l.afterPanelSentinel.current)?void 0:o.contains)&&a.call(o,i))&&i.focus({preventScroll:!0})))}:void 0,tabIndex:-1},j=(0,v.l)(),F=(0,s.z)(()=>{let e=w.current;e&&(0,x.E)(j.current,{[v.N.Forwards]:()=>{var t;(0,S.jA)(e,S.TO.First)===S.fE.Error&&(null==(t=l.afterPanelSentinel.current)||t.focus())},[v.N.Backwards]:()=>{var e;null==(e=l.button)||e.focus({preventScroll:!0})}})}),Z=(0,s.z)(()=>{let e=w.current;e&&(0,x.E)(j.current,{[v.N.Forwards]:()=>{var e;if(!l.button)return;let t=(0,S.GO)(),n=t.indexOf(l.button),r=t.slice(0,n+1),o=[...t.slice(n+1),...r];for(let t of o.slice())if("true"===t.dataset.headlessuiFocusGuard||null!=(e=l.panel)&&e.contains(t)){let e=o.indexOf(t);-1!==e&&o.splice(e,1)}(0,S.jA)(o,S.TO.First,{sorted:!1})},[v.N.Backwards]:()=>{var t;(0,S.jA)(e,S.TO.Previous)===S.fE.Error&&(null==(t=l.button)||t.focus())}})});return i.createElement(D.Provider,{value:r},I&&f&&i.createElement(y._,{id:g,ref:l.beforePanelSentinel,features:y.A.Focusable,"data-headlessui-focus-guard":!0,as:"button",type:"button",onFocus:F}),(0,k.sY)({mergeRefs:A,ourProps:L,theirProps:a,slot:M,defaultTag:"div",features:B,visible:I,name:"Popover.Panel"}),I&&f&&i.createElement(y._,{id:h,ref:l.afterPanelSentinel,features:y.A.Focusable,"data-headlessui-focus-guard":!0,as:"button",type:"button",onFocus:Z}))}),Group:(0,k.yV)(function(e,t){let n=(0,i.useRef)(null),r=(0,b.T)(n,t),[o,a]=(0,i.useState)([]),l=(0,h.H)(),c=(0,s.z)(e=>{a(t=>{let n=t.indexOf(e);if(-1!==n){let e=t.slice();return e.splice(n,1),e}return t})}),u=(0,s.z)(e=>(a(t=>[...t,e]),()=>c(e))),d=(0,s.z)(()=>{var e;let t=(0,O.r)(n);if(!t)return!1;let r=t.activeElement;return!!(null!=(e=n.current)&&e.contains(r))||o.some(e=>{var n,o;return(null==(n=t.getElementById(e.buttonId.current))?void 0:n.contains(r))||(null==(o=t.getElementById(e.panelId.current))?void 0:o.contains(r))})}),p=(0,s.z)(e=>{for(let t of o)t.buttonId.current!==e&&t.close()}),f=(0,i.useMemo)(()=>({registerPopover:u,unregisterPopover:c,isFocusWithinPopoverGroup:d,closeOthers:p,mainTreeNodeRef:l.mainTreeNodeRef}),[u,c,d,p,l.mainTreeNodeRef]),m=(0,i.useMemo)(()=>({}),[]);return i.createElement(M.Provider,{value:f},(0,k.sY)({ourProps:{ref:r},theirProps:e,slot:m,defaultTag:"div",name:"Popover.Group"}),i.createElement(l.MainTreeNode,null))})});var U=n(70129),z=n(25163);let H=e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor"}),i.createElement("path",{fillRule:"evenodd",d:"M6 2a1 1 0 00-1 1v1H4a2 2 0 00-2 2v10a2 2 0 002 2h12a2 2 0 002-2V6a2 2 0 00-2-2h-1V3a1 1 0 10-2 0v1H7V3a1 1 0 00-1-1zm0 5a1 1 0 000 2h8a1 1 0 100-2H6z",clipRule:"evenodd"}))};var G=n(8903),W=n(49492);function $(){return(0,W.Z)(Date.now())}var V=n(32633),q=n(99250),Y=n(91753),K=n(74416),X=n(50295),Q=n(6976),J=n(13256),ee=n(68309),et=n(84120),en=n(27552);function er(e,t){if((0,en.Z)(2,arguments),!t||"object"!==(0,Q.Z)(t))return new Date(NaN);var n=t.years?(0,ee.Z)(t.years):0,r=t.months?(0,ee.Z)(t.months):0,o=t.weeks?(0,ee.Z)(t.weeks):0,a=t.days?(0,ee.Z)(t.days):0,i=t.hours?(0,ee.Z)(t.hours):0,l=t.minutes?(0,ee.Z)(t.minutes):0,s=t.seconds?(0,ee.Z)(t.seconds):0,c=function(e,t){(0,en.Z)(2,arguments);var n=(0,ee.Z)(t);return(0,et.Z)(e,-n)}(e,r+12*n);return new Date((0,J.Z)(c,a+7*o).getTime()-1e3*(s+60*(l+60*i)))}var eo=n(8053),ea=n(68005),ei=n(22893),el=n(65492);let es=(0,el.fn)("DateRangePicker"),ec=(e,t,n,r)=>{var o;if(n&&(e=null===(o=r.get(n))||void 0===o?void 0:o.from),e)return(0,W.Z)(e&&!t?e:(0,K.Z)([e,t]))},eu=(e,t,n,r)=>{var o,a;if(n&&(e=(0,W.Z)(null!==(a=null===(o=r.get(n))||void 0===o?void 0:o.to)&&void 0!==a?a:$())),e)return(0,W.Z)(e&&!t?e:(0,X.Z)([e,t]))},ed=[{value:"tdy",text:"Today",from:$()},{value:"w",text:"Last 7 days",from:er($(),{days:7})},{value:"t",text:"Last 30 days",from:er($(),{days:30})},{value:"m",text:"Month to Date",from:(0,V.Z)($())},{value:"y",text:"Year to Date",from:(0,eo.Z)($())}],ep=(e,t,n,r)=>{let o=(null==n?void 0:n.code)||"en-US";if(!e&&!t)return"";if(e&&!t)return r?(0,ea.Z)(e,r):e.toLocaleDateString(o,{year:"numeric",month:"short",day:"numeric"});if(e&&t){if(function(e,t){(0,en.Z)(2,arguments);var n=(0,ei.Z)(e),r=(0,ei.Z)(t);return n.getTime()===r.getTime()}(e,t))return r?(0,ea.Z)(e,r):e.toLocaleDateString(o,{year:"numeric",month:"short",day:"numeric"});if(e.getMonth()===t.getMonth()&&e.getFullYear()===t.getFullYear())return r?"".concat((0,ea.Z)(e,r)," - ").concat((0,ea.Z)(t,r)):"".concat(e.toLocaleDateString(o,{month:"short",day:"numeric"})," - \n ").concat(t.getDate(),", ").concat(t.getFullYear());{if(r)return"".concat((0,ea.Z)(e,r)," - ").concat((0,ea.Z)(t,r));let n={year:"numeric",month:"short",day:"numeric"};return"".concat(e.toLocaleDateString(o,n)," - \n ").concat(t.toLocaleDateString(o,n))}}return""};var ef=n(26463);let em=e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:"2.5"},t),i.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M15 19l-7-7 7-7"}))},eg=e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:"2.5"},t),i.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M9 5l7 7-7 7"}))},eh=e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:"2.5"}),i.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M11 19l-7-7 7-7m8 14l-7-7 7-7"}))},eb=e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:"2.5"}),i.createElement("path",{strokeLinecap:"round",strokeLinejoin:"round",d:"M13 5l7 7-7 7M5 5l7 7-7 7"}))};var ev=n(45503),ey=n(71801);n(5);var eE=n(58437),ew=n(54942),eS=n(2898);let ex={xs:{paddingX:"px-2",paddingY:"py-0.5",fontSize:"text-xs"},sm:{paddingX:"px-2.5",paddingY:"py-1",fontSize:"text-sm"},md:{paddingX:"px-3",paddingY:"py-1.5",fontSize:"text-md"},lg:{paddingX:"px-3.5",paddingY:"py-1.5",fontSize:"text-lg"},xl:{paddingX:"px-3.5",paddingY:"py-1.5",fontSize:"text-xl"}},eO={xs:{paddingX:"px-2",paddingY:"py-0.5",fontSize:"text-xs"},sm:{paddingX:"px-2.5",paddingY:"py-0.5",fontSize:"text-sm"},md:{paddingX:"px-3",paddingY:"py-0.5",fontSize:"text-md"},lg:{paddingX:"px-3.5",paddingY:"py-0.5",fontSize:"text-lg"},xl:{paddingX:"px-4",paddingY:"py-1",fontSize:"text-xl"}},ek={xs:{height:"h-4",width:"w-4"},sm:{height:"h-4",width:"w-4"},md:{height:"h-4",width:"w-4"},lg:{height:"h-5",width:"w-5"},xl:{height:"h-6",width:"w-6"}},eC={[ew.wu.Increase]:{bgColor:(0,el.bM)(ew.fr.Emerald,eS.K.background).bgColor,textColor:(0,el.bM)(ew.fr.Emerald,eS.K.text).textColor},[ew.wu.ModerateIncrease]:{bgColor:(0,el.bM)(ew.fr.Emerald,eS.K.background).bgColor,textColor:(0,el.bM)(ew.fr.Emerald,eS.K.text).textColor},[ew.wu.Decrease]:{bgColor:(0,el.bM)(ew.fr.Rose,eS.K.background).bgColor,textColor:(0,el.bM)(ew.fr.Rose,eS.K.text).textColor},[ew.wu.ModerateDecrease]:{bgColor:(0,el.bM)(ew.fr.Rose,eS.K.background).bgColor,textColor:(0,el.bM)(ew.fr.Rose,eS.K.text).textColor},[ew.wu.Unchanged]:{bgColor:(0,el.bM)(ew.fr.Orange,eS.K.background).bgColor,textColor:(0,el.bM)(ew.fr.Orange,eS.K.text).textColor}},eT={[ew.wu.Increase]:e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",width:"24",height:"24"}),i.createElement("path",{fill:"none",d:"M0 0h24v24H0z"}),i.createElement("path",{fill:"currentColor",d:"M13 7.828V20h-2V7.828l-5.364 5.364-1.414-1.414L12 4l7.778 7.778-1.414 1.414L13 7.828z"}))},[ew.wu.ModerateIncrease]:e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",width:"24",height:"24"}),i.createElement("path",{fill:"none",d:"M0 0h24v24H0z"}),i.createElement("path",{fill:"currentColor",d:"M16.004 9.414l-8.607 8.607-1.414-1.414L14.589 8H7.004V6h11v11h-2V9.414z"}))},[ew.wu.Decrease]:e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",width:"24",height:"24"}),i.createElement("path",{fill:"none",d:"M0 0h24v24H0z"}),i.createElement("path",{fill:"currentColor",d:"M13 16.172l5.364-5.364 1.414 1.414L12 20l-7.778-7.778 1.414-1.414L11 16.172V4h2v12.172z"}))},[ew.wu.ModerateDecrease]:e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",width:"24",height:"24"}),i.createElement("path",{fill:"none",d:"M0 0h24v24H0z"}),i.createElement("path",{fill:"currentColor",d:"M14.59 16.004L5.982 7.397l1.414-1.414 8.607 8.606V7.004h2v11h-11v-2z"}))},[ew.wu.Unchanged]:e=>{var t=(0,a._T)(e,[]);return i.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",width:"24",height:"24"}),i.createElement("path",{fill:"none",d:"M0 0h24v24H0z"}),i.createElement("path",{fill:"currentColor",d:"M16.172 11l-5.364-5.364 1.414-1.414L20 12l-7.778 7.778-1.414-1.414L16.172 13H4v-2z"}))}},eA=(0,el.fn)("BadgeDelta");i.forwardRef((e,t)=>{let{deltaType:n=ew.wu.Increase,isIncreasePositive:r=!0,size:o=ew.u8.SM,tooltip:l,children:s,className:c}=e,u=(0,a._T)(e,["deltaType","isIncreasePositive","size","tooltip","children","className"]),d=eT[n],p=(0,el.Fo)(n,r),f=s?eO:ex,{tooltipProps:m,getReferenceProps:g}=(0,eE.l)();return i.createElement("span",Object.assign({ref:(0,el.lq)([t,m.refs.setReference]),className:(0,q.q)(eA("root"),"w-max flex-shrink-0 inline-flex justify-center items-center cursor-default rounded-tremor-full bg-opacity-20 dark:bg-opacity-25",eC[p].bgColor,eC[p].textColor,f[o].paddingX,f[o].paddingY,f[o].fontSize,c)},g,u),i.createElement(eE.Z,Object.assign({text:l},m)),i.createElement(d,{className:(0,q.q)(eA("icon"),"shrink-0",s?(0,q.q)("-ml-1 mr-1.5"):ek[o].height,ek[o].width)}),s?i.createElement("p",{className:(0,q.q)(eA("text"),"text-sm whitespace-nowrap")},s):null)}).displayName="BadgeDelta";var eN=n(61244);let eI=e=>{var{onClick:t,icon:n}=e,r=(0,a._T)(e,["onClick","icon"]);return i.createElement("button",Object.assign({type:"button",className:(0,q.q)("flex items-center justify-center p-1 h-7 w-7 outline-none focus:ring-2 transition duration-100 border border-tremor-border dark:border-dark-tremor-border hover:bg-tremor-background-muted dark:hover:bg-dark-tremor-background-muted rounded-tremor-small focus:border-tremor-brand-subtle select-none dark:focus:border-dark-tremor-brand-subtle focus:ring-tremor-brand-muted dark:focus:ring-dark-tremor-brand-muted text-tremor-content-subtle dark:text-dark-tremor-content-subtle hover:text-tremor-content dark:hover:text-dark-tremor-content")},r),i.createElement(eN.Z,{onClick:t,icon:n,variant:"simple",color:"slate",size:"xs"}))};function eR(e){var{mode:t,defaultMonth:n,selected:r,onSelect:o,locale:l,disabled:s,enableYearNavigation:c,classNames:u,weekStartsOn:d=0}=e,p=(0,a._T)(e,["mode","defaultMonth","selected","onSelect","locale","disabled","enableYearNavigation","classNames","weekStartsOn"]);return i.createElement(ef._W,Object.assign({showOutsideDays:!0,mode:t,defaultMonth:n,selected:r,onSelect:o,locale:l,disabled:s,weekStartsOn:d,classNames:Object.assign({months:"flex flex-col sm:flex-row space-y-4 sm:space-x-4 sm:space-y-0",month:"space-y-4",caption:"flex justify-center pt-2 relative items-center",caption_label:"text-tremor-default text-tremor-content-emphasis dark:text-dark-tremor-content-emphasis font-medium",nav:"space-x-1 flex items-center",nav_button:"flex items-center justify-center p-1 h-7 w-7 outline-none focus:ring-2 transition duration-100 border border-tremor-border dark:border-dark-tremor-border hover:bg-tremor-background-muted dark:hover:bg-dark-tremor-background-muted rounded-tremor-small focus:border-tremor-brand-subtle dark:focus:border-dark-tremor-brand-subtle focus:ring-tremor-brand-muted dark:focus:ring-dark-tremor-brand-muted text-tremor-content-subtle dark:text-dark-tremor-content-subtle hover:text-tremor-content dark:hover:text-dark-tremor-content",nav_button_previous:"absolute left-1",nav_button_next:"absolute right-1",table:"w-full border-collapse space-y-1",head_row:"flex",head_cell:"w-9 font-normal text-center text-tremor-content-subtle dark:text-dark-tremor-content-subtle",row:"flex w-full mt-0.5",cell:"text-center p-0 relative focus-within:relative text-tremor-default text-tremor-content-emphasis dark:text-dark-tremor-content-emphasis",day:"h-9 w-9 p-0 hover:bg-tremor-background-subtle dark:hover:bg-dark-tremor-background-subtle outline-tremor-brand dark:outline-dark-tremor-brand rounded-tremor-default",day_today:"font-bold",day_selected:"aria-selected:bg-tremor-background-emphasis aria-selected:text-tremor-content-inverted dark:aria-selected:bg-dark-tremor-background-emphasis dark:aria-selected:text-dark-tremor-content-inverted ",day_disabled:"text-tremor-content-subtle dark:text-dark-tremor-content-subtle disabled:hover:bg-transparent",day_outside:"text-tremor-content-subtle dark:text-dark-tremor-content-subtle"},u),components:{IconLeft:e=>{var t=(0,a._T)(e,[]);return i.createElement(em,Object.assign({className:"h-4 w-4"},t))},IconRight:e=>{var t=(0,a._T)(e,[]);return i.createElement(eg,Object.assign({className:"h-4 w-4"},t))},Caption:e=>{var t=(0,a._T)(e,[]);let{goToMonth:n,nextMonth:r,previousMonth:o,currentMonth:s}=(0,ef.HJ)();return i.createElement("div",{className:"flex justify-between items-center"},i.createElement("div",{className:"flex items-center space-x-1"},c&&i.createElement(eI,{onClick:()=>s&&n((0,ev.Z)(s,-1)),icon:eh}),i.createElement(eI,{onClick:()=>o&&n(o),icon:em})),i.createElement(ey.Z,{className:"text-tremor-default tabular-nums capitalize text-tremor-content-emphasis dark:text-dark-tremor-content-emphasis font-medium"},(0,ea.Z)(t.displayMonth,"LLLL yyy",{locale:l})),i.createElement("div",{className:"flex items-center space-x-1"},i.createElement(eI,{onClick:()=>r&&n(r),icon:eg}),c&&i.createElement(eI,{onClick:()=>s&&n((0,ev.Z)(s,1)),icon:eb})))}}},p))}eR.displayName="DateRangePicker",n(95093);var e_=n(27166),eP=n(82985),eM=n(46457);let eL=$(),eD=i.forwardRef((e,t)=>{var n,r;let{value:o,defaultValue:l,onValueChange:s,enableSelect:c=!0,minDate:u,maxDate:d,placeholder:p="Select range",selectPlaceholder:f="Select range",disabled:m=!1,locale:g=eP.Z,enableClear:h=!0,displayFormat:b,children:v,className:y,enableYearNavigation:E=!1,weekStartsOn:w=0,disabledDates:S}=e,x=(0,a._T)(e,["value","defaultValue","onValueChange","enableSelect","minDate","maxDate","placeholder","selectPlaceholder","disabled","locale","enableClear","displayFormat","children","className","enableYearNavigation","weekStartsOn","disabledDates"]),[O,k]=(0,eM.Z)(l,o),[C,T]=(0,i.useState)(!1),[A,N]=(0,i.useState)(!1),I=(0,i.useMemo)(()=>{let e=[];return u&&e.push({before:u}),d&&e.push({after:d}),[...e,...null!=S?S:[]]},[u,d,S]),R=(0,i.useMemo)(()=>{let e=new Map;return v?i.Children.forEach(v,t=>{var n;e.set(t.props.value,{text:null!==(n=(0,Y.qg)(t))&&void 0!==n?n:t.props.value,from:t.props.from,to:t.props.to})}):ed.forEach(t=>{e.set(t.value,{text:t.text,from:t.from,to:eL})}),e},[v]),_=(0,i.useMemo)(()=>{if(v)return(0,Y.sl)(v);let e=new Map;return ed.forEach(t=>e.set(t.value,t.text)),e},[v]),P=(null==O?void 0:O.selectValue)||"",M=ec(null==O?void 0:O.from,u,P,R),L=eu(null==O?void 0:O.to,d,P,R),D=M||L?ep(M,L,g,b):p,j=(0,V.Z)(null!==(r=null!==(n=null!=L?L:M)&&void 0!==n?n:d)&&void 0!==r?r:eL),F=h&&!m;return i.createElement("div",Object.assign({ref:t,className:(0,q.q)("w-full min-w-[10rem] relative flex justify-between text-tremor-default max-w-sm shadow-tremor-input dark:shadow-dark-tremor-input rounded-tremor-default",y)},x),i.createElement(Z,{as:"div",className:(0,q.q)("w-full",c?"rounded-l-tremor-default":"rounded-tremor-default",C&&"ring-2 ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted z-10")},i.createElement("div",{className:"relative w-full"},i.createElement(Z.Button,{onFocus:()=>T(!0),onBlur:()=>T(!1),disabled:m,className:(0,q.q)("w-full outline-none text-left whitespace-nowrap truncate focus:ring-2 transition duration-100 rounded-l-tremor-default flex flex-nowrap border pl-3 py-2","rounded-l-tremor-default border-tremor-border text-tremor-content-emphasis focus:border-tremor-brand-subtle focus:ring-tremor-brand-muted","dark:border-dark-tremor-border dark:text-dark-tremor-content-emphasis dark:focus:border-dark-tremor-brand-subtle dark:focus:ring-dark-tremor-brand-muted",c?"rounded-l-tremor-default":"rounded-tremor-default",F?"pr-8":"pr-4",(0,Y.um)((0,Y.Uh)(M||L),m))},i.createElement(H,{className:(0,q.q)(es("calendarIcon"),"flex-none shrink-0 h-5 w-5 -ml-0.5 mr-2","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle"),"aria-hidden":"true"}),i.createElement("p",{className:"truncate"},D)),F&&M?i.createElement("button",{type:"button",className:(0,q.q)("absolute outline-none inset-y-0 right-0 flex items-center transition duration-100 mr-4"),onClick:e=>{e.preventDefault(),null==s||s({}),k({})}},i.createElement(G.Z,{className:(0,q.q)(es("clearIcon"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})):null),i.createElement(U.u,{className:"absolute z-10 min-w-min left-0",enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},i.createElement(Z.Panel,{focus:!0,className:(0,q.q)("divide-y overflow-y-auto outline-none rounded-tremor-default p-3 border my-1","bg-tremor-background border-tremor-border divide-tremor-border shadow-tremor-dropdown","dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border dark:shadow-dark-tremor-dropdown")},i.createElement(eR,Object.assign({mode:"range",showOutsideDays:!0,defaultMonth:j,selected:{from:M,to:L},onSelect:e=>{null==s||s({from:null==e?void 0:e.from,to:null==e?void 0:e.to}),k({from:null==e?void 0:e.from,to:null==e?void 0:e.to})},locale:g,disabled:I,enableYearNavigation:E,classNames:{day_range_middle:(0,q.q)("!rounded-none aria-selected:!bg-tremor-background-subtle aria-selected:dark:!bg-dark-tremor-background-subtle aria-selected:!text-tremor-content aria-selected:dark:!bg-dark-tremor-background-subtle"),day_range_start:"rounded-r-none rounded-l-tremor-small aria-selected:text-tremor-brand-inverted dark:aria-selected:text-dark-tremor-brand-inverted",day_range_end:"rounded-l-none rounded-r-tremor-small aria-selected:text-tremor-brand-inverted dark:aria-selected:text-dark-tremor-brand-inverted"},weekStartsOn:w},e))))),c&&i.createElement(z.R,{as:"div",className:(0,q.q)("w-48 -ml-px rounded-r-tremor-default",A&&"ring-2 ring-tremor-brand-muted dark:ring-dark-tremor-brand-muted z-10"),value:P,onChange:e=>{let{from:t,to:n}=R.get(e),r=null!=n?n:eL;null==s||s({from:t,to:r,selectValue:e}),k({from:t,to:r,selectValue:e})},disabled:m},e=>{var t;let{value:n}=e;return i.createElement(i.Fragment,null,i.createElement(z.R.Button,{onFocus:()=>N(!0),onBlur:()=>N(!1),className:(0,q.q)("w-full outline-none text-left whitespace-nowrap truncate rounded-r-tremor-default transition duration-100 border px-4 py-2","border-tremor-border shadow-tremor-input text-tremor-content-emphasis focus:border-tremor-brand-subtle","dark:border-dark-tremor-border dark:shadow-dark-tremor-input dark:text-dark-tremor-content-emphasis dark:focus:border-dark-tremor-brand-subtle",(0,Y.um)((0,Y.Uh)(n),m))},n&&null!==(t=_.get(n))&&void 0!==t?t:f),i.createElement(U.u,{className:"absolute z-10 w-full inset-x-0 right-0",enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},i.createElement(z.R.Options,{className:(0,q.q)("divide-y overflow-y-auto outline-none border my-1","shadow-tremor-dropdown bg-tremor-background border-tremor-border divide-tremor-border rounded-tremor-default","dark:shadow-dark-tremor-dropdown dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border")},null!=v?v:ed.map(e=>i.createElement(e_.Z,{key:e.value,value:e.value},e.text)))))}))});eD.displayName="DateRangePicker"},47047:function(e,t,n){n.d(t,{Z:function(){return b}});var r=n(69703),o=n(64090);n(50027),n(18174),n(21871);var a=n(41213),i=n(46457),l=n(54518);let s=e=>{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({},t,{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 20 20",fill:"currentColor"}),o.createElement("path",{fillRule:"evenodd",d:"M8 4a4 4 0 100 8 4 4 0 000-8zM2 8a6 6 0 1110.89 3.476l4.817 4.817a1 1 0 01-1.414 1.414l-4.816-4.816A6 6 0 012 8z",clipRule:"evenodd"}))};var c=n(8903),u=n(25163),d=n(70129);let p=e=>{var t=(0,r._T)(e,[]);return o.createElement("svg",Object.assign({xmlns:"http://www.w3.org/2000/svg",width:"100%",height:"100%",fill:"none",viewBox:"0 0 24 24",stroke:"currentColor",strokeWidth:"2",strokeLinecap:"round",strokeLinejoin:"round"},t),o.createElement("line",{x1:"18",y1:"6",x2:"6",y2:"18"}),o.createElement("line",{x1:"6",y1:"6",x2:"18",y2:"18"}))};var f=n(99250),m=n(65492),g=n(91753);let h=(0,m.fn)("MultiSelect"),b=o.forwardRef((e,t)=>{let{defaultValue:n,value:m,onValueChange:b,placeholder:v="Select...",placeholderSearch:y="Search",disabled:E=!1,icon:w,children:S,className:x}=e,O=(0,r._T)(e,["defaultValue","value","onValueChange","placeholder","placeholderSearch","disabled","icon","children","className"]),[k,C]=(0,i.Z)(n,m),{reactElementChildren:T,optionsAvailable:A}=(0,o.useMemo)(()=>{let e=o.Children.toArray(S).filter(o.isValidElement);return{reactElementChildren:e,optionsAvailable:(0,g.n0)("",e)}},[S]),[N,I]=(0,o.useState)(""),R=(null!=k?k:[]).length>0,_=(0,o.useMemo)(()=>N?(0,g.n0)(N,T):A,[N,T,A]),P=()=>{I("")};return o.createElement(u.R,Object.assign({as:"div",ref:t,defaultValue:k,value:k,onChange:e=>{null==b||b(e),C(e)},disabled:E,className:(0,f.q)("w-full min-w-[10rem] relative text-tremor-default",x)},O,{multiple:!0}),e=>{let{value:t}=e;return o.createElement(o.Fragment,null,o.createElement(u.R.Button,{className:(0,f.q)("w-full outline-none text-left whitespace-nowrap truncate rounded-tremor-default focus:ring-2 transition duration-100 border pr-8 py-1.5","border-tremor-border shadow-tremor-input focus:border-tremor-brand-subtle focus:ring-tremor-brand-muted","dark:border-dark-tremor-border dark:shadow-dark-tremor-input dark:focus:border-dark-tremor-brand-subtle dark:focus:ring-dark-tremor-brand-muted",w?"p-10 -ml-0.5":"pl-3",(0,g.um)(t.length>0,E))},w&&o.createElement("span",{className:(0,f.q)("absolute inset-y-0 left-0 flex items-center ml-px pl-2.5")},o.createElement(w,{className:(0,f.q)(h("Icon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),o.createElement("div",{className:"h-6 flex items-center"},t.length>0?o.createElement("div",{className:"flex flex-nowrap overflow-x-scroll [&::-webkit-scrollbar]:hidden [scrollbar-width:none] gap-x-1 mr-5 -ml-1.5 relative"},A.filter(e=>t.includes(e.props.value)).map((e,n)=>{var r;return o.createElement("div",{key:n,className:(0,f.q)("max-w-[100px] lg:max-w-[200px] flex justify-center items-center pl-2 pr-1.5 py-1 font-medium","rounded-tremor-small","bg-tremor-background-muted dark:bg-dark-tremor-background-muted","bg-tremor-background-subtle dark:bg-dark-tremor-background-subtle","text-tremor-content-default dark:text-dark-tremor-content-default","text-tremor-content-emphasis dark:text-dark-tremor-content-emphasis")},o.createElement("div",{className:"text-xs truncate "},null!==(r=e.props.children)&&void 0!==r?r:e.props.value),o.createElement("div",{onClick:n=>{n.preventDefault();let r=t.filter(t=>t!==e.props.value);null==b||b(r),C(r)}},o.createElement(p,{className:(0,f.q)(h("clearIconItem"),"cursor-pointer rounded-tremor-full w-3.5 h-3.5 ml-2","text-tremor-content-subtle hover:text-tremor-content","dark:text-dark-tremor-content-subtle dark:hover:text-tremor-content")})))})):o.createElement("span",null,v)),o.createElement("span",{className:(0,f.q)("absolute inset-y-0 right-0 flex items-center mr-2.5")},o.createElement(l.Z,{className:(0,f.q)(h("arrowDownIcon"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}))),R&&!E?o.createElement("button",{type:"button",className:(0,f.q)("absolute inset-y-0 right-0 flex items-center mr-8"),onClick:e=>{e.preventDefault(),C([]),null==b||b([])}},o.createElement(c.Z,{className:(0,f.q)(h("clearIconAllItems"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})):null,o.createElement(d.u,{className:"absolute z-10 w-full",enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},o.createElement(u.R.Options,{className:(0,f.q)("divide-y overflow-y-auto outline-none rounded-tremor-default max-h-[228px] left-0 border my-1","bg-tremor-background border-tremor-border divide-tremor-border shadow-tremor-dropdown","dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border dark:shadow-dark-tremor-dropdown")},o.createElement("div",{className:(0,f.q)("flex items-center w-full px-2.5","bg-tremor-background-muted","dark:bg-dark-tremor-background-muted")},o.createElement("span",null,o.createElement(s,{className:(0,f.q)("flex-none w-4 h-4 mr-2","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),o.createElement("input",{name:"search",type:"input",autoComplete:"off",placeholder:y,className:(0,f.q)("w-full focus:outline-none focus:ring-none bg-transparent text-tremor-default py-2","text-tremor-content-emphasis","dark:text-dark-tremor-content-emphasis"),onKeyDown:e=>{"Space"===e.code&&""!==e.target.value&&e.stopPropagation()},onChange:e=>I(e.target.value),value:N})),o.createElement(a.Z.Provider,Object.assign({},{onBlur:{handleResetSearch:P}},{value:{selectedValue:t}}),_))))})});b.displayName="MultiSelect"},76628:function(e,t,n){n.d(t,{Z:function(){return u}});var r=n(69703);n(50027),n(18174),n(21871);var o=n(41213),a=n(64090),i=n(99250),l=n(65492),s=n(25163);let c=(0,l.fn)("MultiSelectItem"),u=a.forwardRef((e,t)=>{let{value:n,className:u,children:d}=e,p=(0,r._T)(e,["value","className","children"]),{selectedValue:f}=(0,a.useContext)(o.Z),m=(0,l.NZ)(n,f);return a.createElement(s.R.Option,Object.assign({className:(0,i.q)(c("root"),"flex justify-start items-center cursor-default text-tremor-default p-2.5","ui-active:bg-tremor-background-muted ui-active:text-tremor-content-strong ui-selected:text-tremor-content-strong text-tremor-content-emphasis","dark:ui-active:bg-dark-tremor-background-muted dark:ui-active:text-dark-tremor-content-strong dark:ui-selected:text-dark-tremor-content-strong dark:ui-selected:bg-dark-tremor-background-muted dark:text-dark-tremor-content-emphasis",u),ref:t,key:n,value:n},p),a.createElement("input",{type:"checkbox",className:(0,i.q)(c("checkbox"),"flex-none focus:ring-none focus:outline-none cursor-pointer mr-2.5","accent-tremor-brand","dark:accent-dark-tremor-brand"),checked:m,readOnly:!0}),a.createElement("span",{className:"whitespace-nowrap truncate"},null!=d?d:n))});u.displayName="MultiSelectItem"},95093:function(e,t,n){n.d(t,{Z:function(){return m}});var r=n(69703),o=n(64090),a=n(54518),i=n(8903),l=n(99250),s=n(65492),c=n(91753),u=n(25163),d=n(70129),p=n(46457);let f=(0,s.fn)("Select"),m=o.forwardRef((e,t)=>{let{defaultValue:n,value:s,onValueChange:m,placeholder:g="Select...",disabled:h=!1,icon:b,enableClear:v=!0,children:y,className:E}=e,w=(0,r._T)(e,["defaultValue","value","onValueChange","placeholder","disabled","icon","enableClear","children","className"]),[S,x]=(0,p.Z)(n,s),O=(0,o.useMemo)(()=>{let e=o.Children.toArray(y).filter(o.isValidElement);return(0,c.sl)(e)},[y]);return o.createElement(u.R,Object.assign({as:"div",ref:t,defaultValue:S,value:S,onChange:e=>{null==m||m(e),x(e)},disabled:h,className:(0,l.q)("w-full min-w-[10rem] relative text-tremor-default",E)},w),e=>{var t;let{value:n}=e;return o.createElement(o.Fragment,null,o.createElement(u.R.Button,{className:(0,l.q)("w-full outline-none text-left whitespace-nowrap truncate rounded-tremor-default focus:ring-2 transition duration-100 border pr-8 py-2","border-tremor-border shadow-tremor-input focus:border-tremor-brand-subtle focus:ring-tremor-brand-muted","dark:border-dark-tremor-border dark:shadow-dark-tremor-input dark:focus:border-dark-tremor-brand-subtle dark:focus:ring-dark-tremor-brand-muted",b?"p-10 -ml-0.5":"pl-3",(0,c.um)((0,c.Uh)(n),h))},b&&o.createElement("span",{className:(0,l.q)("absolute inset-y-0 left-0 flex items-center ml-px pl-2.5")},o.createElement(b,{className:(0,l.q)(f("Icon"),"flex-none h-5 w-5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})),o.createElement("span",{className:"w-[90%] block truncate"},n&&null!==(t=O.get(n))&&void 0!==t?t:g),o.createElement("span",{className:(0,l.q)("absolute inset-y-0 right-0 flex items-center mr-3")},o.createElement(a.Z,{className:(0,l.q)(f("arrowDownIcon"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}))),v&&S?o.createElement("button",{type:"button",className:(0,l.q)("absolute inset-y-0 right-0 flex items-center mr-8"),onClick:e=>{e.preventDefault(),x(""),null==m||m("")}},o.createElement(i.Z,{className:(0,l.q)(f("clearIcon"),"flex-none h-4 w-4","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")})):null,o.createElement(d.u,{className:"absolute z-10 w-full",enter:"transition ease duration-100 transform",enterFrom:"opacity-0 -translate-y-4",enterTo:"opacity-100 translate-y-0",leave:"transition ease duration-100 transform",leaveFrom:"opacity-100 translate-y-0",leaveTo:"opacity-0 -translate-y-4"},o.createElement(u.R.Options,{className:(0,l.q)("divide-y overflow-y-auto outline-none rounded-tremor-default max-h-[228px] left-0 border my-1","bg-tremor-background border-tremor-border divide-tremor-border shadow-tremor-dropdown","dark:bg-dark-tremor-background dark:border-dark-tremor-border dark:divide-dark-tremor-border dark:shadow-dark-tremor-dropdown")},y)))})});m.displayName="Select"},27166:function(e,t,n){n.d(t,{Z:function(){return s}});var r=n(69703),o=n(64090),a=n(25163),i=n(99250);let l=(0,n(65492).fn)("SelectItem"),s=o.forwardRef((e,t)=>{let{value:n,icon:s,className:c,children:u}=e,d=(0,r._T)(e,["value","icon","className","children"]);return o.createElement(a.R.Option,Object.assign({className:(0,i.q)(l("root"),"flex justify-start items-center cursor-default text-tremor-default px-2.5 py-2.5","ui-active:bg-tremor-background-muted ui-active:text-tremor-content-strong ui-selected:text-tremor-content-strong ui-selected:bg-tremor-background-muted text-tremor-content-emphasis","dark:ui-active:bg-dark-tremor-background-muted dark:ui-active:text-dark-tremor-content-strong dark:ui-selected:text-dark-tremor-content-strong dark:ui-selected:bg-dark-tremor-background-muted dark:text-dark-tremor-content-emphasis",c),ref:t,key:n,value:n},d),s&&o.createElement(s,{className:(0,i.q)(l("icon"),"flex-none w-5 h-5 mr-1.5","text-tremor-content-subtle","dark:text-dark-tremor-content-subtle")}),o.createElement("span",{className:"whitespace-nowrap truncate"},null!=u?u:n))});s.displayName="SelectItem"},12224:function(e,t,n){n.d(t,{Z:function(){return N}});var r=n(69703),o=n(64090),a=n(83891),i=n(20044),l=n(10641),s=n(92381),c=n(71454),u=n(36601),d=n(37700),p=n(84152),f=n(34797),m=n(18318),g=n(71014),h=n(67409),b=n(39790);let v=(0,o.createContext)(null),y=Object.assign((0,m.yV)(function(e,t){let n=(0,s.M)(),{id:r="headlessui-label-".concat(n),passive:a=!1,...i}=e,l=function e(){let t=(0,o.useContext)(v);if(null===t){let t=Error("You used a