From 09874cc83fb8ce9b73524668f5df617822df4d96 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 16:33:03 +0530 Subject: [PATCH 01/11] (v0) add ContentPolicyViolationError --- litellm/__init__.py | 1 + litellm/exceptions.py | 15 +++++++++++++++ litellm/utils.py | 12 ++++++++++++ 3 files changed, 28 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index f848dd32435..7e96d10aa71 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -544,6 +544,7 @@ from .exceptions import ( ServiceUnavailableError, OpenAIError, ContextWindowExceededError, + ContentPolicyViolationError, BudgetExceededError, APIError, Timeout, diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 3898a568301..4f9629e71c8 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -108,6 +108,21 @@ class ContextWindowExceededError(BadRequestError): # type: ignore ) # Call the base class constructor with the parameters it needs +class ContentPolicyViolationError(BadRequestError): # type: ignore + # Error code: 400 - {'error': {'code': 'content_policy_violation', 'message': 'Your request was rejected as a result of our safety system. Image descriptions generated from your prompt may contain text that is not allowed by our safety system. If you believe this was done in error, your request may succeed if retried, or by adjusting your prompt.', 'param': None, 'type': 'invalid_request_error'}} + def __init__(self, message, model, llm_provider, response: httpx.Response): + self.status_code = 400 + self.message = message + self.model = model + self.llm_provider = llm_provider + super().__init__( + message=self.message, + model=self.model, # type: ignore + llm_provider=self.llm_provider, # type: ignore + response=response, + ) # Call the base class constructor with the parameters it needs + + class ServiceUnavailableError(APIStatusError): # type: ignore def __init__(self, message, llm_provider, model, response: httpx.Response): self.status_code = 503 diff --git a/litellm/utils.py b/litellm/utils.py index 4520bee6219..11e0c028ec1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -59,6 +59,7 @@ from .exceptions import ( ServiceUnavailableError, OpenAIError, ContextWindowExceededError, + ContentPolicyViolationError, Timeout, APIConnectionError, APIError, @@ -5559,6 +5560,17 @@ def exception_type( model=model, response=original_exception.response, ) + elif ( + "invalid_request_error" in error_str + and "content_policy_violation" in error_str + ): + exception_mapping_worked = True + raise Con( + message=f"OpenAIException - {original_exception.message}", + llm_provider="openai", + model=model, + response=original_exception.response, + ) elif hasattr(original_exception, "status_code"): exception_mapping_worked = True if original_exception.status_code == 401: From cf98343eb5af79ffde0e812eea674a4b8443973d Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 16:34:20 +0530 Subject: [PATCH 02/11] (test) content policy violation error --- litellm/tests/test_exceptions.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/litellm/tests/test_exceptions.py b/litellm/tests/test_exceptions.py index 1cb599206b4..82f52fa015d 100644 --- a/litellm/tests/test_exceptions.py +++ b/litellm/tests/test_exceptions.py @@ -352,6 +352,24 @@ def test_completion_mistral_exception(): # test_completion_mistral_exception() +def test_content_policy_exceptionimage_generation_openai(): + try: + litellm.set_verbose = True + response = litellm.image_generation( + prompt="a very bad prompt", model="dall-e-3" + ) + print(f"response: {response}") + assert len(response.data) > 0 + except litellm.ContentPolicyViolationError as e: + print("caught a content policy violation error! Passed") + pass + except Exception as e: + pytest.fail(f"An exception occurred - {str(e)}") + + +# test_image_generation_openai() + + # # test_invalid_request_error(model="command-nightly") # # Test 3: Rate Limit Errors # def test_model_call(model): From 9da61bdf3148061a27538ee83f677567739ee272 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 16:53:15 +0530 Subject: [PATCH 03/11] (fix) ContentPolicyViolationError --- litellm/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 11e0c028ec1..ecbaa9e56f4 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5551,10 +5551,10 @@ def exception_type( ) elif ( "invalid_request_error" in error_str - and "Incorrect API key provided" not in error_str + and "content_policy_violation" in error_str ): exception_mapping_worked = True - raise BadRequestError( + raise ContentPolicyViolationError( message=f"OpenAIException - {original_exception.message}", llm_provider="openai", model=model, @@ -5562,10 +5562,10 @@ def exception_type( ) elif ( "invalid_request_error" in error_str - and "content_policy_violation" in error_str + and "Incorrect API key provided" not in error_str ): exception_mapping_worked = True - raise Con( + raise BadRequestError( message=f"OpenAIException - {original_exception.message}", llm_provider="openai", model=model, From f0c10377cf4dc759f5c0ff1fe8e990861b1edf27 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 16:53:57 +0530 Subject: [PATCH 04/11] (test) ContentPolicyViolationError --- litellm/tests/test_exceptions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/litellm/tests/test_exceptions.py b/litellm/tests/test_exceptions.py index 82f52fa015d..998e3eb9b94 100644 --- a/litellm/tests/test_exceptions.py +++ b/litellm/tests/test_exceptions.py @@ -354,9 +354,10 @@ def test_completion_mistral_exception(): def test_content_policy_exceptionimage_generation_openai(): try: + # this is ony a test - we needed some way to invoke the exception :( litellm.set_verbose = True response = litellm.image_generation( - prompt="a very bad prompt", model="dall-e-3" + prompt="where do i buy lethal drugs from", model="dall-e-3" ) print(f"response: {response}") assert len(response.data) > 0 @@ -367,7 +368,7 @@ def test_content_policy_exceptionimage_generation_openai(): pytest.fail(f"An exception occurred - {str(e)}") -# test_image_generation_openai() +# test_content_policy_exceptionimage_generation_openai() # # test_invalid_request_error(model="command-nightly") From 170ae74118f975f5f77b41ad9fd9974072cd8735 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 16:54:47 +0530 Subject: [PATCH 05/11] (feat) add exception mapping for litellm.image_generation --- litellm/main.py | 329 +++++++++++++++++++++++++----------------------- 1 file changed, 173 insertions(+), 156 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 59cbbab3c28..4978c79f1af 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1117,7 +1117,7 @@ def completion( acompletion=acompletion, logging_obj=logging, custom_prompt_dict=custom_prompt_dict, - timeout=timeout + timeout=timeout, ) if ( "stream" in optional_params @@ -2838,158 +2838,167 @@ def image_generation( Currently supports just Azure + OpenAI. """ - aimg_generation = kwargs.get("aimg_generation", False) - litellm_call_id = kwargs.get("litellm_call_id", None) - logger_fn = kwargs.get("logger_fn", None) - proxy_server_request = kwargs.get("proxy_server_request", None) - model_info = kwargs.get("model_info", None) - metadata = kwargs.get("metadata", {}) + try: + aimg_generation = kwargs.get("aimg_generation", False) + litellm_call_id = kwargs.get("litellm_call_id", None) + logger_fn = kwargs.get("logger_fn", None) + proxy_server_request = kwargs.get("proxy_server_request", None) + model_info = kwargs.get("model_info", None) + metadata = kwargs.get("metadata", {}) - model_response = litellm.utils.ImageResponse() - if model is not None or custom_llm_provider is not None: - model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) # type: ignore - else: - model = "dall-e-2" - custom_llm_provider = "openai" # default to dall-e-2 on openai - openai_params = [ - "user", - "request_timeout", - "api_base", - "api_version", - "api_key", - "deployment_id", - "organization", - "base_url", - "default_headers", - "timeout", - "max_retries", - "n", - "quality", - "size", - "style", - ] - litellm_params = [ - "metadata", - "aimg_generation", - "caching", - "mock_response", - "api_key", - "api_version", - "api_base", - "force_timeout", - "logger_fn", - "verbose", - "custom_llm_provider", - "litellm_logging_obj", - "litellm_call_id", - "use_client", - "id", - "fallbacks", - "azure", - "headers", - "model_list", - "num_retries", - "context_window_fallback_dict", - "roles", - "final_prompt_value", - "bos_token", - "eos_token", - "request_timeout", - "complete_response", - "self", - "client", - "rpm", - "tpm", - "input_cost_per_token", - "output_cost_per_token", - "hf_model_name", - "proxy_server_request", - "model_info", - "preset_cache_key", - "caching_groups", - "ttl", - "cache", - ] - default_params = openai_params + litellm_params - non_default_params = { - k: v for k, v in kwargs.items() if k not in default_params - } # model-specific params - pass them straight to the model/provider - optional_params = get_optional_params_image_gen( - n=n, - quality=quality, - response_format=response_format, - size=size, - style=style, - user=user, - custom_llm_provider=custom_llm_provider, - **non_default_params, - ) - logging = litellm_logging_obj - logging.update_environment_variables( - model=model, - user=user, - optional_params=optional_params, - litellm_params={ - "timeout": timeout, - "azure": False, - "litellm_call_id": litellm_call_id, - "logger_fn": logger_fn, - "proxy_server_request": proxy_server_request, - "model_info": model_info, - "metadata": metadata, - "preset_cache_key": None, - "stream_response": {}, - }, - ) - - if custom_llm_provider == "azure": - # azure configs - api_type = get_secret("AZURE_API_TYPE") or "azure" - - api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") - - api_version = ( - api_version or litellm.api_version or get_secret("AZURE_API_VERSION") + model_response = litellm.utils.ImageResponse() + if model is not None or custom_llm_provider is not None: + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider, api_base=api_base) # type: ignore + else: + model = "dall-e-2" + custom_llm_provider = "openai" # default to dall-e-2 on openai + openai_params = [ + "user", + "request_timeout", + "api_base", + "api_version", + "api_key", + "deployment_id", + "organization", + "base_url", + "default_headers", + "timeout", + "max_retries", + "n", + "quality", + "size", + "style", + ] + litellm_params = [ + "metadata", + "aimg_generation", + "caching", + "mock_response", + "api_key", + "api_version", + "api_base", + "force_timeout", + "logger_fn", + "verbose", + "custom_llm_provider", + "litellm_logging_obj", + "litellm_call_id", + "use_client", + "id", + "fallbacks", + "azure", + "headers", + "model_list", + "num_retries", + "context_window_fallback_dict", + "roles", + "final_prompt_value", + "bos_token", + "eos_token", + "request_timeout", + "complete_response", + "self", + "client", + "rpm", + "tpm", + "input_cost_per_token", + "output_cost_per_token", + "hf_model_name", + "proxy_server_request", + "model_info", + "preset_cache_key", + "caching_groups", + "ttl", + "cache", + ] + default_params = openai_params + litellm_params + non_default_params = { + k: v for k, v in kwargs.items() if k not in default_params + } # model-specific params - pass them straight to the model/provider + optional_params = get_optional_params_image_gen( + n=n, + quality=quality, + response_format=response_format, + size=size, + style=style, + user=user, + custom_llm_provider=custom_llm_provider, + **non_default_params, ) - - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret("AZURE_OPENAI_API_KEY") - or get_secret("AZURE_API_KEY") - ) - - azure_ad_token = optional_params.pop("azure_ad_token", None) or get_secret( - "AZURE_AD_TOKEN" - ) - - model_response = azure_chat_completions.image_generation( + logging = litellm_logging_obj + logging.update_environment_variables( model=model, - prompt=prompt, - timeout=timeout, - api_key=api_key, - api_base=api_base, - logging_obj=litellm_logging_obj, + user=user, optional_params=optional_params, - model_response=model_response, - api_version=api_version, - aimg_generation=aimg_generation, - ) - elif custom_llm_provider == "openai": - model_response = openai_chat_completions.image_generation( - model=model, - prompt=prompt, - timeout=timeout, - api_key=api_key, - api_base=api_base, - logging_obj=litellm_logging_obj, - optional_params=optional_params, - model_response=model_response, - aimg_generation=aimg_generation, + litellm_params={ + "timeout": timeout, + "azure": False, + "litellm_call_id": litellm_call_id, + "logger_fn": logger_fn, + "proxy_server_request": proxy_server_request, + "model_info": model_info, + "metadata": metadata, + "preset_cache_key": None, + "stream_response": {}, + }, ) - return model_response + if custom_llm_provider == "azure": + # azure configs + api_type = get_secret("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") + + api_version = ( + api_version or litellm.api_version or get_secret("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.pop("azure_ad_token", None) or get_secret( + "AZURE_AD_TOKEN" + ) + + model_response = azure_chat_completions.image_generation( + model=model, + prompt=prompt, + timeout=timeout, + api_key=api_key, + api_base=api_base, + logging_obj=litellm_logging_obj, + optional_params=optional_params, + model_response=model_response, + api_version=api_version, + aimg_generation=aimg_generation, + ) + elif custom_llm_provider == "openai": + model_response = openai_chat_completions.image_generation( + model=model, + prompt=prompt, + timeout=timeout, + api_key=api_key, + api_base=api_base, + logging_obj=litellm_logging_obj, + optional_params=optional_params, + model_response=model_response, + aimg_generation=aimg_generation, + ) + + return model_response + except Exception as e: + ## Map to OpenAI Exception + raise exception_type( + model=model, + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=locals(), + ) ##### Health Endpoints ####################### @@ -3114,7 +3123,8 @@ def config_completion(**kwargs): "No config path set, please set a config path using `litellm.config_path = 'path/to/config.json'`" ) -def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List]=None): + +def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List] = None): id = chunks[0]["id"] object = chunks[0]["object"] created = chunks[0]["created"] @@ -3131,23 +3141,27 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List]= "system_fingerprint": system_fingerprint, "choices": [ { - "text": None, - "index": 0, - "logprobs": logprobs, - "finish_reason": finish_reason + "text": None, + "index": 0, + "logprobs": logprobs, + "finish_reason": finish_reason, } ], "usage": { "prompt_tokens": None, "completion_tokens": None, - "total_tokens": None - } + "total_tokens": None, + }, } content_list = [] for chunk in chunks: choices = chunk["choices"] for choice in choices: - if choice is not None and hasattr(choice, "text") and choice.get("text") is not None: + if ( + choice is not None + and hasattr(choice, "text") + and choice.get("text") is not None + ): _choice = choice.get("text") content_list.append(_choice) @@ -3179,13 +3193,16 @@ def stream_chunk_builder_text_completion(chunks: list, messages: Optional[List]= ) return response + def stream_chunk_builder(chunks: list, messages: Optional[list] = None): id = chunks[0]["id"] object = chunks[0]["object"] created = chunks[0]["created"] model = chunks[0]["model"] system_fingerprint = chunks[0].get("system_fingerprint", None) - if isinstance(chunks[0]["choices"][0], litellm.utils.TextChoices): # route to the text completion logic + if isinstance( + chunks[0]["choices"][0], litellm.utils.TextChoices + ): # route to the text completion logic return stream_chunk_builder_text_completion(chunks=chunks, messages=messages) role = chunks[0]["choices"][0]["delta"]["role"] finish_reason = chunks[-1]["choices"][0]["finish_reason"] From 186fc4614de3bb4fba7ab8a3b2b0aea0cf3c9210 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 16:58:09 +0530 Subject: [PATCH 06/11] (feat) add ContentPolicyViolationError for azure --- litellm/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/litellm/utils.py b/litellm/utils.py index ecbaa9e56f4..1cf1ae9f882 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6509,6 +6509,17 @@ def exception_type( model=model, response=original_exception.response, ) + elif ( + "invalid_request_error" in error_str + and "content_policy_violation" in error_str + ): + exception_mapping_worked = True + raise ContentPolicyViolationError( + message=f"AzureException - {original_exception.message}", + llm_provider="azure", + model=model, + response=original_exception.response, + ) elif "invalid_request_error" in error_str: exception_mapping_worked = True raise BadRequestError( From c0b56b65758511e7e66c9e12ab3cb019bcdc4ee0 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 17:04:04 +0530 Subject: [PATCH 07/11] (test) catch litellm.ContentPolicyViolationError --- litellm/tests/test_image_generation.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/litellm/tests/test_image_generation.py b/litellm/tests/test_image_generation.py index 973ec29bb87..3c792f80229 100644 --- a/litellm/tests/test_image_generation.py +++ b/litellm/tests/test_image_generation.py @@ -19,7 +19,7 @@ import litellm def test_image_generation_openai(): - try: + try: litellm.set_verbose = True response = litellm.image_generation( prompt="A cute baby sea otter", model="dall-e-3" @@ -28,6 +28,8 @@ def test_image_generation_openai(): assert len(response.data) > 0 except litellm.RateLimitError as e: pass + except litellm.ContentPolicyViolationError: + pass # OpenAI randomly raises these errors - skip when they occur except Exception as e: pytest.fail(f"An exception occurred - {str(e)}") @@ -36,22 +38,27 @@ def test_image_generation_openai(): def test_image_generation_azure(): - try: + try: response = litellm.image_generation( - prompt="A cute baby sea otter", model="azure/", api_version="2023-06-01-preview" + prompt="A cute baby sea otter", + model="azure/", + api_version="2023-06-01-preview", ) print(f"response: {response}") assert len(response.data) > 0 except litellm.RateLimitError as e: pass + except litellm.ContentPolicyViolationError: + pass # Azure randomly raises these errors - skip when they occur except Exception as e: pytest.fail(f"An exception occurred - {str(e)}") + # test_image_generation_azure() def test_image_generation_azure_dall_e_3(): - try: + try: litellm.set_verbose = True response = litellm.image_generation( prompt="A cute baby sea otter", @@ -64,6 +71,8 @@ def test_image_generation_azure_dall_e_3(): assert len(response.data) > 0 except litellm.RateLimitError as e: pass + except litellm.ContentPolicyViolationError: + pass # OpenAI randomly raises these errors - skip when they occur except Exception as e: pytest.fail(f"An exception occurred - {str(e)}") @@ -71,7 +80,7 @@ def test_image_generation_azure_dall_e_3(): # test_image_generation_azure_dall_e_3() @pytest.mark.asyncio async def test_async_image_generation_openai(): - try: + try: response = litellm.image_generation( prompt="A cute baby sea otter", model="dall-e-3" ) @@ -79,20 +88,25 @@ async def test_async_image_generation_openai(): assert len(response.data) > 0 except litellm.RateLimitError as e: pass + except litellm.ContentPolicyViolationError: + pass # openai randomly raises these errors - skip when they occur except Exception as e: pytest.fail(f"An exception occurred - {str(e)}") + # asyncio.run(test_async_image_generation_openai()) @pytest.mark.asyncio async def test_async_image_generation_azure(): - try: + try: response = await litellm.aimage_generation( prompt="A cute baby sea otter", model="azure/dall-e-3-test" ) print(f"response: {response}") except litellm.RateLimitError as e: pass + except litellm.ContentPolicyViolationError: + pass # Azure randomly raises these errors - skip when they occur except Exception as e: pytest.fail(f"An exception occurred - {str(e)}") From e8987d9b677b892a2ebd29104e9dc304b01c1a61 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 17:06:35 +0530 Subject: [PATCH 08/11] (docs) remove outdated tutorials --- docs/my-website/sidebars.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 45a979921bb..2d36b02d2c3 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -134,8 +134,6 @@ const sidebars = { label: 'Tutorials', items: [ 'tutorials/azure_openai', - "tutorials/lm_evaluation_harness", - "tutorials/eval_suites", 'tutorials/oobabooga', "tutorials/gradio_integration", 'tutorials/huggingface_codellama', From cc07ba1d6a9685c5b88e4d11a82e1810a9ab8adc Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 17:09:41 +0530 Subject: [PATCH 09/11] (docs) move tutorials --- docs/my-website/sidebars.js | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 2d36b02d2c3..3c6e0011dbb 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -129,24 +129,6 @@ const sidebars = { "secret", "completion/token_usage", "load_test", - { - type: 'category', - label: 'Tutorials', - items: [ - 'tutorials/azure_openai', - 'tutorials/oobabooga', - "tutorials/gradio_integration", - 'tutorials/huggingface_codellama', - 'tutorials/huggingface_tutorial', - 'tutorials/TogetherAI_liteLLM', - 'tutorials/finetuned_chat_gpt', - 'tutorials/sagemaker_llms', - 'tutorials/text_completion', - "tutorials/first_playground", - 'tutorials/compare_llms', - "tutorials/model_fallbacks", - ], - }, { type: "category", label: "Logging & Observability", @@ -168,6 +150,23 @@ const sidebars = { ], }, "caching/redis_cache", + { + type: 'category', + label: 'Tutorials', + items: [ + 'tutorials/azure_openai', + 'tutorials/oobabooga', + "tutorials/gradio_integration", + 'tutorials/huggingface_codellama', + 'tutorials/huggingface_tutorial', + 'tutorials/TogetherAI_liteLLM', + 'tutorials/finetuned_chat_gpt', + 'tutorials/sagemaker_llms', + 'tutorials/text_completion', + "tutorials/first_playground", + "tutorials/model_fallbacks", + ], + }, { type: "category", label: "LangChain, LlamaIndex Integration", From 248e5f3d9219b11f429ef6074e6c84517fec6b29 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 17:13:06 +0530 Subject: [PATCH 10/11] (chore) remove deprecated completion_with_config() tests --- docs/my-website/docs/completion/config.md | 49 --------- litellm/__init__.py | 1 - litellm/tests/test_config.py | 118 -------------------- litellm/utils.py | 127 ---------------------- 4 files changed, 295 deletions(-) delete mode 100644 docs/my-website/docs/completion/config.md delete mode 100644 litellm/tests/test_config.py diff --git a/docs/my-website/docs/completion/config.md b/docs/my-website/docs/completion/config.md deleted file mode 100644 index 44ff6e8a70f..00000000000 --- a/docs/my-website/docs/completion/config.md +++ /dev/null @@ -1,49 +0,0 @@ -# Model Config - -Model-specific changes can make our code complicated, making it harder to debug errors. Use model configs to simplify this. - -### usage - -Handling prompt logic. Different models have different context windows. Use `adapt_to_prompt_size` to select the right model for the prompt (in case the current model is too small). - - -```python -from litellm import completion_with_config -import os - -config = { - "available_models": ["gpt-3.5-turbo", "claude-instant-1", "gpt-3.5-turbo-16k"], - "adapt_to_prompt_size": True, # 👈 key change -} - -# set env var -os.environ["OPENAI_API_KEY"] = "your-api-key" -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - - -sample_text = "how does a court case get to the Supreme Court?" * 1000 -messages = [{"content": sample_text, "role": "user"}] -response = completion_with_config(model="gpt-3.5-turbo", messages=messages, config=config) -``` - -[**See Code**](https://github.com/BerriAI/litellm/blob/30724d9e51cdc2c3e0eb063271b4f171bc01b382/litellm/utils.py#L2783) - -### Complete Config Structure - -```python -config = { - "default_fallback_models": # [Optional] List of model names to try if a call fails - "available_models": # [Optional] List of all possible models you could call - "adapt_to_prompt_size": # [Optional] True/False - if you want to select model based on prompt size (will pick from available_models) - "model": { - "model-name": { - "needs_moderation": # [Optional] True/False - if you want to call openai moderations endpoint before making completion call. Will raise exception, if flagged. - "error_handling": { - "error-type": { # One of the errors listed here - https://docs.litellm.ai/docs/exception_mapping#custom-mapping-list - "fallback_model": "" # str, name of the model it should try instead, when that error occurs - } - } - } - } -} -``` \ No newline at end of file diff --git a/litellm/__init__.py b/litellm/__init__.py index f848dd32435..018b8bb148e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -500,7 +500,6 @@ from .utils import ( validate_environment, check_valid_key, get_llm_provider, - completion_with_config, register_model, encode, decode, diff --git a/litellm/tests/test_config.py b/litellm/tests/test_config.py deleted file mode 100644 index 69e37cf87cf..00000000000 --- a/litellm/tests/test_config.py +++ /dev/null @@ -1,118 +0,0 @@ -import sys, os -import traceback -from dotenv import load_dotenv - -load_dotenv() -import os - -sys.path.insert( - 0, os.path.abspath("../..") -) # Adds the parent directory to the system path -import pytest -import litellm -from litellm import completion_with_config - -config = { - "default_fallback_models": ["gpt-3.5-turbo", "claude-instant-1", "j2-ultra"], - "model": { - "claude-instant-1": {"needs_moderation": True}, - "gpt-3.5-turbo": { - "error_handling": { - "ContextWindowExceededError": {"fallback_model": "gpt-3.5-turbo-16k"} - } - }, - }, -} - - -def test_config_context_window_exceeded(): - try: - sample_text = "how does a court case get to the Supreme Court?" * 1000 - messages = [{"content": sample_text, "role": "user"}] - response = completion_with_config( - model="gpt-3.5-turbo", messages=messages, config=config - ) - print(response) - except Exception as e: - print(f"Exception: {e}") - pytest.fail(f"An exception occurred: {e}") - - -# test_config_context_window_exceeded() - - -def test_config_context_moderation(): - try: - messages = [{"role": "user", "content": "I want to kill them."}] - response = completion_with_config( - model="claude-instant-1", messages=messages, config=config - ) - print(response) - except Exception as e: - print(f"Exception: {e}") - pytest.fail(f"An exception occurred: {e}") - - -# test_config_context_moderation() - - -def test_config_context_default_fallback(): - try: - messages = [{"role": "user", "content": "Hey, how's it going?"}] - response = completion_with_config( - model="claude-instant-1", - messages=messages, - config=config, - api_key="bad-key", - ) - print(response) - except Exception as e: - print(f"Exception: {e}") - pytest.fail(f"An exception occurred: {e}") - - -# test_config_context_default_fallback() - - -config = { - "default_fallback_models": ["gpt-3.5-turbo", "claude-instant-1", "j2-ultra"], - "available_models": [ - "gpt-3.5-turbo", - "gpt-3.5-turbo-0301", - "gpt-3.5-turbo-0613", - "gpt-4", - "gpt-4-0314", - "gpt-4-0613", - "j2-ultra", - "command-nightly", - "togethercomputer/llama-2-70b-chat", - "chat-bison", - "chat-bison@001", - "claude-2", - ], - "adapt_to_prompt_size": True, # type: ignore - "model": { - "claude-instant-1": {"needs_moderation": True}, - "gpt-3.5-turbo": { - "error_handling": { - "ContextWindowExceededError": {"fallback_model": "gpt-3.5-turbo-16k"} - } - }, - }, -} - - -def test_config_context_adapt_to_prompt(): - try: - sample_text = "how does a court case get to the Supreme Court?" * 1000 - messages = [{"content": sample_text, "role": "user"}] - response = completion_with_config( - model="gpt-3.5-turbo", messages=messages, config=config - ) - print(response) - except Exception as e: - print(f"Exception: {e}") - pytest.fail(f"An exception occurred: {e}") - - -test_config_context_adapt_to_prompt() diff --git a/litellm/utils.py b/litellm/utils.py index 4520bee6219..47c5695bb84 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7843,133 +7843,6 @@ def read_config_args(config_path) -> dict: ########## experimental completion variants ############################ -def completion_with_config(config: Union[dict, str], **kwargs): - """ - Generate a litellm.completion() using a config dict and all supported completion args - - Example config; - config = { - "default_fallback_models": # [Optional] List of model names to try if a call fails - "available_models": # [Optional] List of all possible models you could call - "adapt_to_prompt_size": # [Optional] True/False - if you want to select model based on prompt size (will pick from available_models) - "model": { - "model-name": { - "needs_moderation": # [Optional] True/False - if you want to call openai moderations endpoint before making completion call. Will raise exception, if flagged. - "error_handling": { - "error-type": { # One of the errors listed here - https://docs.litellm.ai/docs/exception_mapping#custom-mapping-list - "fallback_model": "" # str, name of the model it should try instead, when that error occurs - } - } - } - } - } - - Parameters: - config (Union[dict, str]): A configuration for litellm - **kwargs: Additional keyword arguments for litellm.completion - - Returns: - litellm.ModelResponse: A ModelResponse with the generated completion - - """ - if config is not None: - if isinstance(config, str): - config = read_config_args(config) - elif isinstance(config, dict): - config = config - else: - raise Exception("Config path must be a string or a dictionary.") - else: - raise Exception("Config path not passed in.") - - if config is None: - raise Exception("No completion config in the config file") - - models_with_config = config["model"].keys() - model = kwargs["model"] - messages = kwargs["messages"] - - ## completion config - fallback_models = config.get("default_fallback_models", None) - available_models = config.get("available_models", None) - adapt_to_prompt_size = config.get("adapt_to_prompt_size", False) - trim_messages_flag = config.get("trim_messages", False) - prompt_larger_than_model = False - max_model = model - try: - max_tokens = litellm.get_max_tokens(model)["max_tokens"] - except: - max_tokens = 2048 # assume curr model's max window is 2048 tokens - if adapt_to_prompt_size: - ## Pick model based on token window - prompt_tokens = litellm.token_counter( - model="gpt-3.5-turbo", - text="".join(message["content"] for message in messages), - ) - try: - curr_max_tokens = litellm.get_max_tokens(model)["max_tokens"] - except: - curr_max_tokens = 2048 - if curr_max_tokens < prompt_tokens: - prompt_larger_than_model = True - for available_model in available_models: - try: - curr_max_tokens = litellm.get_max_tokens(available_model)[ - "max_tokens" - ] - if curr_max_tokens > max_tokens: - max_tokens = curr_max_tokens - max_model = available_model - if curr_max_tokens > prompt_tokens: - model = available_model - prompt_larger_than_model = False - except: - continue - if prompt_larger_than_model: - messages = trim_messages(messages=messages, model=max_model) - kwargs["messages"] = messages - - kwargs["model"] = model - try: - if model in models_with_config: - ## Moderation check - if config["model"][model].get("needs_moderation"): - input = " ".join(message["content"] for message in messages) - response = litellm.moderation(input=input) - flagged = response["results"][0]["flagged"] - if flagged: - raise Exception("This response was flagged as inappropriate") - - ## Model-specific Error Handling - error_handling = None - if config["model"][model].get("error_handling"): - error_handling = config["model"][model]["error_handling"] - - try: - response = litellm.completion(**kwargs) - return response - except Exception as e: - exception_name = type(e).__name__ - fallback_model = None - if error_handling and exception_name in error_handling: - error_handler = error_handling[exception_name] - # either switch model or api key - fallback_model = error_handler.get("fallback_model", None) - if fallback_model: - kwargs["model"] = fallback_model - return litellm.completion(**kwargs) - raise e - else: - return litellm.completion(**kwargs) - except Exception as e: - if fallback_models: - model = fallback_models.pop(0) - return completion_with_fallbacks( - model=model, messages=messages, fallbacks=fallback_models - ) - raise e - - def completion_with_fallbacks(**kwargs): nested_kwargs = kwargs.pop("kwargs", {}) response = None From 75d44fad6665e0c7cb1a2423de45756c3b8ad5f6 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 9 Jan 2024 17:16:35 +0530 Subject: [PATCH 11/11] (docs) add ContentPolicyViolationError --- docs/my-website/docs/exception_mapping.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/exception_mapping.md b/docs/my-website/docs/exception_mapping.md index c6c9bb25506..db17fb093e5 100644 --- a/docs/my-website/docs/exception_mapping.md +++ b/docs/my-website/docs/exception_mapping.md @@ -12,6 +12,7 @@ LiteLLM maps exceptions across all providers to their OpenAI counterparts. | 429 | RateLimitError | | >=500 | InternalServerError | | N/A | ContextWindowExceededError| +| 400 | ContentPolicyViolationError| | N/A | APIConnectionError |