From ecf6d22dc2a88ac43b0575d1e19d76a466517c87 Mon Sep 17 00:00:00 2001 From: ProphetJeremy Date: Mon, 13 Jan 2025 14:27:27 +0100 Subject: [PATCH 01/25] (docs) Update vertex.md old code example Complete imports Remove invalid parameter `disable_atributon` --- docs/my-website/docs/providers/vertex.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index cb8c031c062..0c741b64834 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -404,14 +404,16 @@ curl http://localhost:4000/v1/chat/completions \ If this was your initial VertexAI Grounding code, ```python -import vertexai +import vertexai +from vertexai.generative_models import GenerativeModel, GenerationConfig, Tool, grounding + vertexai.init(project=project_id, location="us-central1") model = GenerativeModel("gemini-1.5-flash-001") # Use Google Search for grounding -tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval(disable_attributon=False)) +tool = Tool.from_google_search_retrieval(grounding.GoogleSearchRetrieval()) prompt = "When is the next total solar eclipse in US?" response = model.generate_content( @@ -428,7 +430,7 @@ print(response) then, this is what it looks like now ```python -from litellm import completion +from litellm import completion # !gcloud auth application-default login - run this to add vertex credentials to your env From 0c30909fe9b1dcbd263bc3132f1c2886aab3642a Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Sat, 8 Feb 2025 12:31:01 +0900 Subject: [PATCH 02/25] Reimplement methods required for triton streaming --- .../llms/triton/completion/transformation.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 0cd69400637..9b100ff1f88 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -3,7 +3,7 @@ Translates from OpenAI's `/v1/chat/completions` endpoint to Triton's `/generate` """ import json -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, AsyncIterator, Dict, Iterator, List, Literal, Optional, Union from httpx import Headers, Response @@ -52,6 +52,17 @@ class TritonConfig(BaseConfig): ) -> Dict: return {"Content-Type": "application/json"} + def get_complete_url( + self, + api_base: str, + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + if stream: + return api_base + "_stream" + return api_base + def get_supported_openai_params(self, model: str) -> List: return ["max_tokens", "max_completion_tokens"] @@ -149,6 +160,18 @@ class TritonConfig(BaseConfig): else: raise ValueError(f"Invalid Triton API base: {api_base}") + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse], + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + return TritonResponseIterator( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) + class TritonGenerateConfig(TritonConfig): """ From 268702722504ec2f4bf8f72f7bb15cb6da6843b3 Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:40:56 +0900 Subject: [PATCH 03/25] Apply streaming-related transformations only for generate config --- .../llms/triton/completion/transformation.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index 9b100ff1f88..b09f7b04443 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -52,17 +52,6 @@ class TritonConfig(BaseConfig): ) -> Dict: return {"Content-Type": "application/json"} - def get_complete_url( - self, - api_base: str, - model: str, - optional_params: dict, - stream: Optional[bool] = None, - ) -> str: - if stream: - return api_base + "_stream" - return api_base - def get_supported_openai_params(self, model: str) -> List: return ["max_tokens", "max_completion_tokens"] @@ -178,6 +167,17 @@ class TritonGenerateConfig(TritonConfig): Transformations for triton /generate endpoint (This is a trtllm model) """ + def get_complete_url( + self, + api_base: str, + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + if stream: + return api_base + "_stream" + return api_base + def transform_request( self, model: str, @@ -227,7 +227,7 @@ class TritonGenerateConfig(TritonConfig): return model_response -class TritonInferConfig(TritonGenerateConfig): +class TritonInferConfig(TritonConfig): """ Transformations for triton /infer endpoint (his is an infer model with a custom model on triton) """ From c1f2ae97c5e3573cbfff173c05cf88ad3b35a249 Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Thu, 13 Feb 2025 15:43:42 +0900 Subject: [PATCH 04/25] Add streaming test --- tests/llm_translation/test_triton.py | 40 +++++++++++++++++++++------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/llm_translation/test_triton.py b/tests/llm_translation/test_triton.py index 0835d09fab6..7e4ba92f23b 100644 --- a/tests/llm_translation/test_triton.py +++ b/tests/llm_translation/test_triton.py @@ -49,16 +49,26 @@ def test_split_embedding_by_shape_fails_with_shape_value_error(): ) -def test_completion_triton_generate_api(): +@pytest.mark.parametrize("stream", [True, False]) +def test_completion_triton_generate_api(stream): try: mock_response = MagicMock() + if stream: + def mock_iter_lines(): + mock_output = ''.join([ + 'data: {"model_name":"ensemble","model_version":"1","sequence_end":false,"sequence_id":0,"sequence_start":false,"text_output":"' + t + '"}\n\n' + for t in ["I", " am", " an", " AI", " assistant"] + ]) + for out in mock_output.split('\n'): + yield out + mock_response.iter_lines = mock_iter_lines + else: + def return_val(): + return { + "text_output": "I am an AI assistant", + } - def return_val(): - return { - "text_output": "I am an AI assistant", - } - - mock_response.json = return_val + mock_response.json = return_val mock_response.status_code = 200 with patch( @@ -71,6 +81,7 @@ def test_completion_triton_generate_api(): max_tokens=10, timeout=5, api_base="http://localhost:8000/generate", + stream=stream, ) # Verify the call was made @@ -81,7 +92,10 @@ def test_completion_triton_generate_api(): call_kwargs = mock_post.call_args.kwargs # Access kwargs directly # Verify URL - assert call_kwargs["url"] == "http://localhost:8000/generate" + if stream: + assert call_kwargs["url"] == "http://localhost:8000/generate_stream" + else: + assert call_kwargs["url"] == "http://localhost:8000/generate" # Parse the request data from the JSON string request_data = json.loads(call_kwargs["data"]) @@ -91,7 +105,15 @@ def test_completion_triton_generate_api(): assert request_data["parameters"]["max_tokens"] == 10 # Verify response - assert response.choices[0].message.content == "I am an AI assistant" + if stream: + tokens = ["I", " am", " an", " AI", " assistant", None] + idx = 0 + for chunk in response: + assert chunk.choices[0].delta.content == tokens[idx] + idx += 1 + assert idx == len(tokens) + else: + assert response.choices[0].message.content == "I am an AI assistant" except Exception as e: print("exception", e) From c62be184c2e9228ad321384f1c385be1ff4f882b Mon Sep 17 00:00:00 2001 From: Minwoo Lee <11580164+minwhoo@users.noreply.github.com> Date: Thu, 13 Feb 2025 16:41:50 +0900 Subject: [PATCH 05/25] Fix get_complete_url --- .../llms/triton/completion/transformation.py | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/litellm/llms/triton/completion/transformation.py b/litellm/llms/triton/completion/transformation.py index b09f7b04443..0a65e216dfe 100644 --- a/litellm/llms/triton/completion/transformation.py +++ b/litellm/llms/triton/completion/transformation.py @@ -67,6 +67,18 @@ class TritonConfig(BaseConfig): optional_params[param] = value return optional_params + def get_complete_url( + self, + api_base: str, + model: str, + optional_params: dict, + stream: Optional[bool] = None, + ) -> str: + llm_type = self._get_triton_llm_type(api_base) + if llm_type == "generate" and stream: + return api_base + "_stream" + return api_base + def transform_response( self, model: str, @@ -167,17 +179,6 @@ class TritonGenerateConfig(TritonConfig): Transformations for triton /generate endpoint (This is a trtllm model) """ - def get_complete_url( - self, - api_base: str, - model: str, - optional_params: dict, - stream: Optional[bool] = None, - ) -> str: - if stream: - return api_base + "_stream" - return api_base - def transform_request( self, model: str, From e3455cd0451d051cd388e346773a930d4ad865d2 Mon Sep 17 00:00:00 2001 From: Nitin Patel Date: Mon, 24 Feb 2025 01:00:07 +0530 Subject: [PATCH 06/25] fix missing comma --- litellm/llms/perplexity/chat/transformation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/llms/perplexity/chat/transformation.py b/litellm/llms/perplexity/chat/transformation.py index 8f71cc153f3..dab64283ec2 100644 --- a/litellm/llms/perplexity/chat/transformation.py +++ b/litellm/llms/perplexity/chat/transformation.py @@ -37,6 +37,7 @@ class PerplexityChatConfig(OpenAIGPTConfig): "response_format", "stream", "temperature", - "top_p" "max_retries", + "top_p", + "max_retries", "extra_headers", ] From 4b8db4ec347f1950453c4c94d8ecaff794fbd076 Mon Sep 17 00:00:00 2001 From: Yazan Agha-Schrader Date: Mon, 24 Feb 2025 11:18:18 +0100 Subject: [PATCH 07/25] Update model_prices_and_context_window.json fix mistral/mistral-small from 1$/3$ per million tokens to -> 0.1$/0.3$ per million tokens cave: azure_ai and bedrock still show 1$/3$ for input/output cost per million - i dont have knowledge about azure and bedrock prices, but looks like wrong values as well. **please check** --- model_prices_and_context_window.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5e8d9353ad9..932393c2615 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1943,8 +1943,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000003, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat", @@ -1955,8 +1955,8 @@ "max_tokens": 8191, "max_input_tokens": 32000, "max_output_tokens": 8191, - "input_cost_per_token": 0.000001, - "output_cost_per_token": 0.000003, + "input_cost_per_token": 0.0000001, + "output_cost_per_token": 0.0000003, "litellm_provider": "mistral", "supports_function_calling": true, "mode": "chat", From c40d45ae093489c599bc023912e7fcf5f5dadcc8 Mon Sep 17 00:00:00 2001 From: Vivek Aditya Date: Wed, 26 Feb 2025 21:00:56 +0530 Subject: [PATCH 08/25] Added tags to additional keys that can be sent to athina --- docs/my-website/docs/observability/athina_integration.md | 1 + litellm/integrations/athina.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/observability/athina_integration.md b/docs/my-website/docs/observability/athina_integration.md index 4994d553c62..2e2141169a7 100644 --- a/docs/my-website/docs/observability/athina_integration.md +++ b/docs/my-website/docs/observability/athina_integration.md @@ -78,6 +78,7 @@ Following are the allowed fields in metadata, their types, and their description * `context: Optional[Union[dict, str]]` - This is the context used as information for the prompt. For RAG applications, this is the "retrieved" data. You may log context as a string or as an object (dictionary). * `expected_response: Optional[str]` - This is the reference response to compare against for evaluation purposes. This is useful for segmenting inference calls by expected response. * `user_query: Optional[str]` - This is the user's query. For conversational applications, this is the user's last message. +* `tags: Optional[list]` - This is a list of tags. This is useful for segmenting inference calls by tags. * `custom_attributes: Optional[dict]` - This is a dictionary of custom attributes. This is useful for additional information about the inference. ## Using a self hosted deployment of Athina diff --git a/litellm/integrations/athina.py b/litellm/integrations/athina.py index 754e980c2ad..f416b30f8e5 100644 --- a/litellm/integrations/athina.py +++ b/litellm/integrations/athina.py @@ -23,6 +23,7 @@ class AthinaLogger: "context", "expected_response", "user_query", + "tags", "custom_attributes", ] @@ -78,10 +79,12 @@ class AthinaLogger: # Add additional metadata keys metadata = kwargs.get("litellm_params", {}).get("metadata", {}) if metadata: + print("additional_keys", self.additional_keys) for key in self.additional_keys: + print("key", key) if key in metadata: + print("key is being added", key) data[key] = metadata[key] - response = litellm.module_level_client.post( self.athina_logging_url, headers=self.headers, From ed75dd61c2d982895e5caef25bb7ce58d1248536 Mon Sep 17 00:00:00 2001 From: Vivek Aditya Date: Fri, 28 Feb 2025 21:48:13 +0530 Subject: [PATCH 09/25] Removed prints and added unit tests --- .../docs/observability/athina_integration.md | 2 + litellm/integrations/athina.py | 5 +- tests/litellm/integrations/test_athina.py | 207 ++++++++++++++++++ 3 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 tests/litellm/integrations/test_athina.py diff --git a/docs/my-website/docs/observability/athina_integration.md b/docs/my-website/docs/observability/athina_integration.md index 2e2141169a7..ba93ea4c980 100644 --- a/docs/my-website/docs/observability/athina_integration.md +++ b/docs/my-website/docs/observability/athina_integration.md @@ -79,6 +79,8 @@ Following are the allowed fields in metadata, their types, and their description * `expected_response: Optional[str]` - This is the reference response to compare against for evaluation purposes. This is useful for segmenting inference calls by expected response. * `user_query: Optional[str]` - This is the user's query. For conversational applications, this is the user's last message. * `tags: Optional[list]` - This is a list of tags. This is useful for segmenting inference calls by tags. +* `user_feedback: Optional[str]` - The end user’s feedback. +* `model_options: Optional[dict]` - This is a dictionary of model options. This is useful for getting insights into how model behavior affects your end users. * `custom_attributes: Optional[dict]` - This is a dictionary of custom attributes. This is useful for additional information about the inference. ## Using a self hosted deployment of Athina diff --git a/litellm/integrations/athina.py b/litellm/integrations/athina.py index f416b30f8e5..705dc11f1d3 100644 --- a/litellm/integrations/athina.py +++ b/litellm/integrations/athina.py @@ -24,6 +24,8 @@ class AthinaLogger: "expected_response", "user_query", "tags", + "user_feedback", + "model_options", "custom_attributes", ] @@ -79,11 +81,8 @@ class AthinaLogger: # Add additional metadata keys metadata = kwargs.get("litellm_params", {}).get("metadata", {}) if metadata: - print("additional_keys", self.additional_keys) for key in self.additional_keys: - print("key", key) if key in metadata: - print("key is being added", key) data[key] = metadata[key] response = litellm.module_level_client.post( self.athina_logging_url, diff --git a/tests/litellm/integrations/test_athina.py b/tests/litellm/integrations/test_athina.py new file mode 100644 index 00000000000..fd660a036ed --- /dev/null +++ b/tests/litellm/integrations/test_athina.py @@ -0,0 +1,207 @@ +import unittest +from unittest.mock import patch, MagicMock, ANY +import json +import datetime +import sys +import os + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system-path + +from litellm.integrations.athina import AthinaLogger + +class TestAthinaLogger(unittest.TestCase): + + def setUp(self): + # Set up environment variables for testing + self.env_patcher = patch.dict('os.environ', { + 'ATHINA_API_KEY': 'test-api-key', + 'ATHINA_BASE_URL': 'https://test.athina.ai' + }) + self.env_patcher.start() + self.logger = AthinaLogger() + + # Setup common test variables + self.start_time = datetime.datetime(2023, 1, 1, 12, 0, 0) + self.end_time = datetime.datetime(2023, 1, 1, 12, 0, 1) + self.print_verbose = MagicMock() + + def tearDown(self): + self.env_patcher.stop() + + def test_init(self): + """Test the initialization of AthinaLogger""" + self.assertEqual(self.logger.athina_api_key, 'test-api-key') + self.assertEqual(self.logger.athina_logging_url, 'https://test.athina.ai/api/v1/log/inference') + self.assertEqual(self.logger.headers, { + 'athina-api-key': 'test-api-key', + 'Content-Type': 'application/json' + }) + + @patch('litellm.module_level_client.post') + def test_log_event_success(self, mock_post): + """Test successful logging of an event""" + # Setup mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = "Success" + mock_post.return_value = mock_response + + # Create test data + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': 'Hello'}], + 'stream': False, + 'litellm_params': { + 'metadata': { + 'environment': 'test-environment', + 'prompt_slug': 'test-prompt', + 'customer_id': 'test-customer', + 'customer_user_id': 'test-user', + 'session_id': 'test-session', + 'external_reference_id': 'test-ext-ref', + 'context': 'test-context', + 'expected_response': 'test-expected', + 'user_query': 'test-query', + 'tags': ['test-tag'], + 'user_feedback': 'test-feedback', + 'model_options': {'test-opt': 'test-val'}, + 'custom_attributes': {'test-attr': 'test-val'} + } + } + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = { + 'id': 'resp-123', + 'choices': [{'message': {'content': 'Hi there'}}], + 'usage': { + 'prompt_tokens': 10, + 'completion_tokens': 5, + 'total_tokens': 15 + } + } + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify the results + mock_post.assert_called_once() + call_args = mock_post.call_args + self.assertEqual(call_args[0][0], 'https://test.athina.ai/api/v1/log/inference') + self.assertEqual(call_args[1]['headers'], self.logger.headers) + + # Parse and verify the sent data + sent_data = json.loads(call_args[1]['data']) + self.assertEqual(sent_data['language_model_id'], 'gpt-4') + self.assertEqual(sent_data['prompt'], kwargs['messages']) + self.assertEqual(sent_data['prompt_tokens'], 10) + self.assertEqual(sent_data['completion_tokens'], 5) + self.assertEqual(sent_data['total_tokens'], 15) + self.assertEqual(sent_data['response_time'], 1000) # 1 second = 1000ms + self.assertEqual(sent_data['customer_id'], 'test-customer') + self.assertEqual(sent_data['session_id'], 'test-session') + self.assertEqual(sent_data['environment'], 'test-environment') + self.assertEqual(sent_data['prompt_slug'], 'test-prompt') + self.assertEqual(sent_data['external_reference_id'], 'test-ext-ref') + self.assertEqual(sent_data['context'], 'test-context') + self.assertEqual(sent_data['expected_response'], 'test-expected') + self.assertEqual(sent_data['user_query'], 'test-query') + self.assertEqual(sent_data['tags'], ['test-tag']) + self.assertEqual(sent_data['user_feedback'], 'test-feedback') + self.assertEqual(sent_data['model_options'], {'test-opt': 'test-val'}) + self.assertEqual(sent_data['custom_attributes'], {'test-attr': 'test-val'}) + # Verify the print_verbose was called + self.print_verbose.assert_called_once_with("Athina Logger Succeeded - Success") + + @patch('litellm.module_level_client.post') + def test_log_event_error_response(self, mock_post): + """Test handling of error response from the API""" + # Setup mock error response + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.text = "Bad Request" + mock_post.return_value = mock_response + + # Create test data + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': 'Hello'}], + 'stream': False + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = { + 'id': 'resp-123', + 'choices': [{'message': {'content': 'Hi there'}}], + 'usage': { + 'prompt_tokens': 10, + 'completion_tokens': 5, + 'total_tokens': 15 + } + } + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify print_verbose was called with error message + self.print_verbose.assert_called_once_with("Athina Logger Error - Bad Request, 400") + + @patch('litellm.module_level_client.post') + def test_log_event_exception(self, mock_post): + """Test handling of exceptions during logging""" + # Setup mock to raise exception + mock_post.side_effect = Exception("Test exception") + + # Create test data + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': 'Hello'}], + 'stream': False + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = {} + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify print_verbose was called with exception info + self.print_verbose.assert_called_once() + self.assertIn("Athina Logger Error - Test exception", self.print_verbose.call_args[0][0]) + + @patch('litellm.module_level_client.post') + def test_log_event_with_tools(self, mock_post): + """Test logging with tools/functions data""" + # Setup mock response + mock_response = MagicMock() + mock_response.status_code = 200 + mock_post.return_value = mock_response + + # Create test data with tools + kwargs = { + 'model': 'gpt-4', + 'messages': [{'role': 'user', 'content': "What's the weather?"}], + 'stream': False, + 'optional_params': { + 'tools': [{'type': 'function', 'function': {'name': 'get_weather'}}] + } + } + + response_obj = MagicMock() + response_obj.model_dump.return_value = { + 'id': 'resp-123', + 'usage': {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15} + } + + # Call the method + self.logger.log_event(kwargs, response_obj, self.start_time, self.end_time, self.print_verbose) + + # Verify the results + sent_data = json.loads(mock_post.call_args[1]['data']) + self.assertEqual(sent_data['tools'], [{'type': 'function', 'function': {'name': 'get_weather'}}]) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 666690c31cc415317cb981cae00b5664bff40f38 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Mar 2025 10:18:03 -0700 Subject: [PATCH 10/25] fix atext_completion --- litellm/main.py | 46 +++++++++++----------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 846a908a8e9..903e0e938b0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3900,42 +3900,18 @@ async def atext_completion( ctx = contextvars.copy_context() func_with_context = partial(ctx.run, func) - _, custom_llm_provider, _, _ = get_llm_provider( - model=model, api_base=kwargs.get("api_base", None) - ) - - if ( - custom_llm_provider == "openai" - or custom_llm_provider == "azure" - or custom_llm_provider == "azure_text" - or custom_llm_provider == "custom_openai" - or custom_llm_provider == "anyscale" - or custom_llm_provider == "mistral" - or custom_llm_provider == "openrouter" - or custom_llm_provider == "deepinfra" - or custom_llm_provider == "perplexity" - or custom_llm_provider == "groq" - or custom_llm_provider == "nvidia_nim" - or custom_llm_provider == "cerebras" - or custom_llm_provider == "sambanova" - or custom_llm_provider == "ai21_chat" - or custom_llm_provider == "ai21" - or custom_llm_provider == "volcengine" - or custom_llm_provider == "text-completion-codestral" - or custom_llm_provider == "deepseek" - or custom_llm_provider == "text-completion-openai" - or custom_llm_provider == "huggingface" - or custom_llm_provider == "ollama" - or custom_llm_provider == "vertex_ai" - or custom_llm_provider in litellm.openai_compatible_providers - ): # currently implemented aiohttp calls for just azure and openai, soon all. - # Await normally - response = await loop.run_in_executor(None, func_with_context) - if asyncio.iscoroutine(response): - response = await response + init_response = await loop.run_in_executor(None, func_with_context) + if isinstance(init_response, dict) or isinstance( + init_response, TextCompletionResponse + ): ## CACHING SCENARIO + if isinstance(init_response, dict): + response = TextCompletionResponse(**init_response) + response = init_response + elif asyncio.iscoroutine(init_response): + response = await init_response else: - # Call the synchronous function using run_in_executor - response = await loop.run_in_executor(None, func_with_context) + response = init_response # type: ignore + if ( kwargs.get("stream", False) is True or isinstance(response, TextCompletionStreamWrapper) From 6d537aec48274e8482fbd46389714bef92e22c41 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Mar 2025 10:36:50 -0700 Subject: [PATCH 11/25] OpenAI_Text --- .../components/add_model/provider_specific_fields.tsx | 3 ++- .../src/components/provider_info_helpers.tsx | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index b3da80c7155..365d75dbad6 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -99,7 +99,8 @@ const ProviderSpecificFields: React.FC = ({ {(selectedProviderEnum === Providers.Azure || selectedProviderEnum === Providers.Azure_AI_Studio || - selectedProviderEnum === Providers.OpenAI_Compatible + selectedProviderEnum === Providers.OpenAI_Compatible || + selectedProviderEnum === Providers.OpenAI_Text_Compatible ) && ( = { OpenAI: "openai", + OpenAI_Text: "text-completion-openai", Azure: "azure", Azure_AI_Studio: "azure_ai", Anthropic: "anthropic", @@ -37,6 +41,7 @@ export const provider_map: Record = { MistralAI: "mistral", Cohere: "cohere_chat", OpenAI_Compatible: "openai", + OpenAI_Text_Compatible: "text-completion-openai", Vertex_AI: "vertex_ai", Databricks: "databricks", xAI: "xai", @@ -53,6 +58,9 @@ export const provider_map: Record = { export const providerLogoMap: Record = { [Providers.OpenAI]: "https://artificialanalysis.ai/img/logos/openai_small.svg", + [Providers.OpenAI_Text]: "https://artificialanalysis.ai/img/logos/openai_small.svg", + [Providers.OpenAI_Text_Compatible]: "https://artificialanalysis.ai/img/logos/openai_small.svg", + [Providers.OpenAI_Compatible]: "https://artificialanalysis.ai/img/logos/openai_small.svg", [Providers.Azure]: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", [Providers.Azure_AI_Studio]: "https://upload.wikimedia.org/wikipedia/commons/a/a8/Microsoft_Azure_Logo.svg", [Providers.Anthropic]: "https://artificialanalysis.ai/img/logos/anthropic_small.svg", @@ -61,7 +69,6 @@ export const providerLogoMap: Record = { [Providers.Groq]: "https://artificialanalysis.ai/img/logos/groq_small.png", [Providers.MistralAI]: "https://artificialanalysis.ai/img/logos/mistral_small.png", [Providers.Cohere]: "https://artificialanalysis.ai/img/logos/cohere_small.png", - [Providers.OpenAI_Compatible]: "https://upload.wikimedia.org/wikipedia/commons/4/4e/OpenAI_Logo.svg", [Providers.Vertex_AI]: "https://artificialanalysis.ai/img/logos/google_small.svg", [Providers.Databricks]: "https://artificialanalysis.ai/img/logos/databricks_small.png", [Providers.Ollama]: "https://artificialanalysis.ai/img/logos/ollama_small.svg", From 51f074682f420e11df5a468c998add1b230a0b4b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 10 Mar 2025 10:40:48 -0700 Subject: [PATCH 12/25] show eu api base on openai + text --- .../src/components/add_model/provider_specific_fields.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx index 365d75dbad6..b7565b0494f 100644 --- a/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx +++ b/ui/litellm-dashboard/src/components/add_model/provider_specific_fields.tsx @@ -23,7 +23,7 @@ const ProviderSpecificFields: React.FC = ({ console.log(`type of selectedProviderEnum: ${typeof selectedProviderEnum}`); return ( <> - {selectedProviderEnum === Providers.OpenAI && ( + {selectedProviderEnum === Providers.OpenAI || selectedProviderEnum === Providers.OpenAI_Text && ( <> Date: Mon, 10 Mar 2025 12:20:37 -0700 Subject: [PATCH 13/25] fix linting error --- ui/litellm-dashboard/src/components/transform_request.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/transform_request.tsx b/ui/litellm-dashboard/src/components/transform_request.tsx index 879132ef50b..5d405df78f9 100644 --- a/ui/litellm-dashboard/src/components/transform_request.tsx +++ b/ui/litellm-dashboard/src/components/transform_request.tsx @@ -156,7 +156,7 @@ ${formattedBody} }}>

Original Request

-

The request you would send to LiteLLM's `/chat/completions` endpoint.

+

The request you would send to LiteLLM's /chat/completions endpoint.