From 23769b3603562f8fb4f55ee745674db6d6cca105 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 13:44:16 -0700 Subject: [PATCH 01/31] rename fine tuning apis --- litellm/fine_tuning/main.py | 10 +++++----- .../{openai_fine_tuning => fine_tuning_apis}/openai.py | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename litellm/llms/{openai_fine_tuning => fine_tuning_apis}/openai.py (100%) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index eb5c7d4a435..8dcfe6300fa 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -17,7 +17,7 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union import httpx import litellm -from litellm.llms.openai_fine_tuning.openai import ( +from litellm.llms.fine_tuning_apis.openai import ( FineTuningJob, FineTuningJobCreate, OpenAIFineTuningAPI, @@ -27,7 +27,7 @@ from litellm.types.router import * from litellm.utils import supports_httpx_timeout ####### ENVIRONMENT VARIABLES ################### -openai_fine_tuning_instance = OpenAIFineTuningAPI() +fine_tuning_apis_instance = OpenAIFineTuningAPI() ################################################# @@ -154,7 +154,7 @@ def create_fine_tuning_job( seed=seed, ) - response = openai_fine_tuning_instance.create_fine_tuning_job( + response = fine_tuning_apis_instance.create_fine_tuning_job( api_base=api_base, api_key=api_key, organization=organization, @@ -275,7 +275,7 @@ def cancel_fine_tuning_job( _is_async = kwargs.pop("acancel_fine_tuning_job", False) is True - response = openai_fine_tuning_instance.cancel_fine_tuning_job( + response = fine_tuning_apis_instance.cancel_fine_tuning_job( api_base=api_base, api_key=api_key, organization=organization, @@ -401,7 +401,7 @@ def list_fine_tuning_jobs( _is_async = kwargs.pop("alist_fine_tuning_jobs", False) is True - response = openai_fine_tuning_instance.list_fine_tuning_jobs( + response = fine_tuning_apis_instance.list_fine_tuning_jobs( api_base=api_base, api_key=api_key, organization=organization, diff --git a/litellm/llms/openai_fine_tuning/openai.py b/litellm/llms/fine_tuning_apis/openai.py similarity index 100% rename from litellm/llms/openai_fine_tuning/openai.py rename to litellm/llms/fine_tuning_apis/openai.py From 73fcac87cb7619baaae00ac2edd14e0c4cf2d467 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 15:42:14 -0700 Subject: [PATCH 02/31] add azure ft test file --- litellm/tests/azure_fine_tune.jsonl | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 litellm/tests/azure_fine_tune.jsonl diff --git a/litellm/tests/azure_fine_tune.jsonl b/litellm/tests/azure_fine_tune.jsonl new file mode 100644 index 00000000000..ef41bd97731 --- /dev/null +++ b/litellm/tests/azure_fine_tune.jsonl @@ -0,0 +1,12 @@ +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "What's the capital of France?"}, {"role": "assistant", "content": "Paris, as if everyone doesn't know that already."}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "Who wrote 'Romeo and Juliet'?"}, {"role": "assistant", "content": "Oh, just some guy named William Shakespeare. Ever heard of him?"}]} +{"messages": [{"role": "system", "content": "Clippy is a factual chatbot that is also sarcastic."}, {"role": "user", "content": "How far is the Moon from Earth?"}, {"role": "assistant", "content": "Around 384,400 kilometers. Give or take a few, like that really matters."}]} \ No newline at end of file From 13362029d8aa1c6798ac892dbdf84292fbed4711 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 15:44:53 -0700 Subject: [PATCH 03/31] add support for fine tuning azure --- litellm/fine_tuning/main.py | 96 +++++++++++++++++++++++++++---------- 1 file changed, 71 insertions(+), 25 deletions(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 8dcfe6300fa..72119185f22 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -17,6 +17,8 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union import httpx import litellm +from litellm import get_secret +from litellm.llms.fine_tuning_apis.azure import AzureOpenAIFineTuningAPI from litellm.llms.fine_tuning_apis.openai import ( FineTuningJob, FineTuningJobCreate, @@ -27,7 +29,8 @@ from litellm.types.router import * from litellm.utils import supports_httpx_timeout ####### ENVIRONMENT VARIABLES ################### -fine_tuning_apis_instance = OpenAIFineTuningAPI() +openai_fine_tuning_apis_instance = OpenAIFineTuningAPI() +azure_fine_tuning_apis_instance = AzureOpenAIFineTuningAPI() ################################################# @@ -39,7 +42,7 @@ async def acreate_fine_tuning_job( validation_file: Optional[str] = None, integrations: Optional[List[str]] = None, seed: Optional[int] = None, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -89,7 +92,7 @@ def create_fine_tuning_job( validation_file: Optional[str] = None, integrations: Optional[List[str]] = None, seed: Optional[int] = None, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -101,7 +104,25 @@ def create_fine_tuning_job( """ try: + _is_async = kwargs.pop("acreate_fine_tuning_job", False) is True optional_params = GenericLiteLLMParams(**kwargs) + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + # OpenAI if custom_llm_provider == "openai": # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -124,25 +145,6 @@ def create_fine_tuning_job( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) - ### TIMEOUT LOGIC ### - timeout = ( - optional_params.timeout or kwargs.get("request_timeout", 600) or 600 - ) - # set timeout for 10 minutes by default - - if ( - timeout is not None - and isinstance(timeout, httpx.Timeout) - and supports_httpx_timeout(custom_llm_provider) == False - ): - read_timeout = timeout.read or 600 - timeout = read_timeout # default 10 min timeout - elif timeout is not None and not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore - elif timeout is None: - timeout = 600.0 - - _is_async = kwargs.pop("acreate_fine_tuning_job", False) is True create_fine_tuning_job_data = FineTuningJobCreate( model=model, @@ -154,7 +156,7 @@ def create_fine_tuning_job( seed=seed, ) - response = fine_tuning_apis_instance.create_fine_tuning_job( + response = openai_fine_tuning_apis_instance.create_fine_tuning_job( api_base=api_base, api_key=api_key, organization=organization, @@ -163,6 +165,50 @@ def create_fine_tuning_job( max_retries=optional_params.max_retries, _is_async=_is_async, ) + # Azure OpenAI + elif custom_llm_provider == "azure": + api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token: Optional[str] = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + create_fine_tuning_job_data = FineTuningJobCreate( + model=model, + training_file=training_file, + hyperparameters=hyperparameters, + suffix=suffix, + validation_file=validation_file, + integrations=integrations, + seed=seed, + ) + + response = azure_fine_tuning_apis_instance.create_fine_tuning_job( + api_base=api_base, + api_key=api_key, + api_version=api_version, + create_fine_tuning_job_data=create_fine_tuning_job_data, + timeout=timeout, + max_retries=optional_params.max_retries, + _is_async=_is_async, + ) else: raise litellm.exceptions.BadRequestError( message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( @@ -275,7 +321,7 @@ def cancel_fine_tuning_job( _is_async = kwargs.pop("acancel_fine_tuning_job", False) is True - response = fine_tuning_apis_instance.cancel_fine_tuning_job( + response = openai_fine_tuning_apis_instance.cancel_fine_tuning_job( api_base=api_base, api_key=api_key, organization=organization, @@ -401,7 +447,7 @@ def list_fine_tuning_jobs( _is_async = kwargs.pop("alist_fine_tuning_jobs", False) is True - response = fine_tuning_apis_instance.list_fine_tuning_jobs( + response = openai_fine_tuning_apis_instance.list_fine_tuning_jobs( api_base=api_base, api_key=api_key, organization=organization, From 63cbb6e6341f2e605dda3dc7823a841a8ac04ca1 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 15:45:15 -0700 Subject: [PATCH 04/31] add azure fine tuning apis --- litellm/llms/fine_tuning_apis/azure.py | 178 +++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 litellm/llms/fine_tuning_apis/azure.py diff --git a/litellm/llms/fine_tuning_apis/azure.py b/litellm/llms/fine_tuning_apis/azure.py new file mode 100644 index 00000000000..6c32e2ac788 --- /dev/null +++ b/litellm/llms/fine_tuning_apis/azure.py @@ -0,0 +1,178 @@ +from typing import Any, Coroutine, Optional, Union + +import httpx +from openai import AsyncAzureOpenAI, AzureOpenAI +from openai.pagination import AsyncCursorPage +from openai.types.fine_tuning import FineTuningJob + +from litellm._logging import verbose_logger +from litellm.llms.base import BaseLLM +from litellm.llms.files_apis.azure import get_azure_openai_client +from litellm.types.llms.openai import FineTuningJobCreate + + +class AzureOpenAIFineTuningAPI(BaseLLM): + """ + AzureOpenAI methods to support for batches + """ + + def __init__(self) -> None: + super().__init__() + + async def acreate_fine_tuning_job( + self, + create_fine_tuning_job_data: FineTuningJobCreate, + openai_client: AsyncAzureOpenAI, + ) -> FineTuningJob: + response = await openai_client.fine_tuning.jobs.create( + **create_fine_tuning_job_data # type: ignore + ) + return response + + def create_fine_tuning_job( + self, + _is_async: bool, + create_fine_tuning_job_data: FineTuningJobCreate, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str] = None, + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + api_version: Optional[str] = None, + ) -> Union[FineTuningJob, Union[Coroutine[Any, Any, FineTuningJob]]]: + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + api_version=api_version, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.acreate_fine_tuning_job( # type: ignore + create_fine_tuning_job_data=create_fine_tuning_job_data, + openai_client=openai_client, + ) + verbose_logger.debug( + "creating fine tuning job, args= %s", create_fine_tuning_job_data + ) + response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) # type: ignore + return response + + async def acancel_fine_tuning_job( + self, + fine_tuning_job_id: str, + openai_client: AsyncAzureOpenAI, + ) -> FineTuningJob: + response = await openai_client.fine_tuning.jobs.cancel( + fine_tuning_job_id=fine_tuning_job_id + ) + return response + + def cancel_fine_tuning_job( + self, + _is_async: bool, + fine_tuning_job_id: str, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + ): + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.acancel_fine_tuning_job( # type: ignore + fine_tuning_job_id=fine_tuning_job_id, + openai_client=openai_client, + ) + verbose_logger.debug("canceling fine tuning job, args= %s", fine_tuning_job_id) + response = openai_client.fine_tuning.jobs.cancel( + fine_tuning_job_id=fine_tuning_job_id + ) + return response + + async def alist_fine_tuning_jobs( + self, + openai_client: AsyncAzureOpenAI, + after: Optional[str] = None, + limit: Optional[int] = None, + ): + response = await openai_client.fine_tuning.jobs.list(after=after, limit=limit) # type: ignore + return response + + def list_fine_tuning_jobs( + self, + _is_async: bool, + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + after: Optional[str] = None, + limit: Optional[int] = None, + ): + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.alist_fine_tuning_jobs( # type: ignore + after=after, + limit=limit, + openai_client=openai_client, + ) + verbose_logger.debug("list fine tuning job, after= %s, limit= %s", after, limit) + response = openai_client.fine_tuning.jobs.list(after=after, limit=limit) # type: ignore + return response + pass From 566dc43d96a678c66604ba3d1a8ee1bcc4e6062e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 15:45:43 -0700 Subject: [PATCH 05/31] add azure files api --- litellm/files/main.py | 88 ++++++--- litellm/llms/files_apis/azure.py | 315 +++++++++++++++++++++++++++++++ 2 files changed, 375 insertions(+), 28 deletions(-) create mode 100644 litellm/llms/files_apis/azure.py diff --git a/litellm/files/main.py b/litellm/files/main.py index 836f22f9677..b3fbd775f61 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -14,7 +14,8 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union import httpx import litellm -from litellm import client +from litellm import client, get_secret +from litellm.llms.files_apis.azure import AzureOpenAIFilesAPI from litellm.llms.openai import FileDeleted, FileObject, OpenAIFilesAPI from litellm.types.llms.openai import ( Batch, @@ -28,6 +29,7 @@ from litellm.utils import supports_httpx_timeout ####### ENVIRONMENT VARIABLES ################### openai_files_instance = OpenAIFilesAPI() +azure_files_instance = AzureOpenAIFilesAPI() ################################################# @@ -402,7 +404,7 @@ def file_list( async def acreate_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -455,7 +457,31 @@ def create_file( LiteLLM Equivalent of POST: POST https://api.openai.com/v1/files """ try: + _is_async = kwargs.pop("acreate_file", False) is True optional_params = GenericLiteLLMParams(**kwargs) + + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _create_file_request = CreateFileRequest( + file=file, + purpose=purpose, + extra_headers=extra_headers, + extra_body=extra_body, + ) if custom_llm_provider == "openai": # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( @@ -477,32 +503,6 @@ def create_file( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) - ### TIMEOUT LOGIC ### - timeout = ( - optional_params.timeout or kwargs.get("request_timeout", 600) or 600 - ) - # set timeout for 10 minutes by default - - if ( - timeout is not None - and isinstance(timeout, httpx.Timeout) - and supports_httpx_timeout(custom_llm_provider) == False - ): - read_timeout = timeout.read or 600 - timeout = read_timeout # default 10 min timeout - elif timeout is not None and not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore - elif timeout is None: - timeout = 600.0 - - _create_file_request = CreateFileRequest( - file=file, - purpose=purpose, - extra_headers=extra_headers, - extra_body=extra_body, - ) - - _is_async = kwargs.pop("acreate_file", False) is True response = openai_files_instance.create_file( _is_async=_is_async, @@ -513,6 +513,38 @@ def create_file( organization=organization, create_file_data=_create_file_request, ) + elif custom_llm_provider == "azure": + api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token: Optional[str] = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_files_instance.create_file( + _is_async=_is_async, + api_base=api_base, + api_key=api_key, + api_version=api_version, + timeout=timeout, + max_retries=optional_params.max_retries, + create_file_data=_create_file_request, + ) else: raise litellm.exceptions.BadRequestError( message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( diff --git a/litellm/llms/files_apis/azure.py b/litellm/llms/files_apis/azure.py new file mode 100644 index 00000000000..d46d9225a4b --- /dev/null +++ b/litellm/llms/files_apis/azure.py @@ -0,0 +1,315 @@ +from typing import Any, Coroutine, Dict, List, Optional, Union + +import httpx +from openai import AsyncAzureOpenAI, AzureOpenAI +from openai.types.file_deleted import FileDeleted + +import litellm +from litellm._logging import verbose_logger +from litellm.llms.base import BaseLLM +from litellm.types.llms.openai import * + + +def get_azure_openai_client( + api_key: Optional[str], + api_base: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + api_version: Optional[str] = None, + organization: Optional[str] = None, + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + _is_async: bool = False, +) -> Optional[Union[AzureOpenAI, AsyncAzureOpenAI]]: + received_args = locals() + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None + if client is None: + data = {} + for k, v in received_args.items(): + if k == "self" or k == "client" or k == "_is_async": + pass + elif k == "api_base" and v is not None: + data["azure_endpoint"] = v + elif v is not None: + data[k] = v + if "api_version" not in data: + data["api_version"] = litellm.AZURE_DEFAULT_API_VERSION + if _is_async is True: + openai_client = AsyncAzureOpenAI(**data) + else: + openai_client = AzureOpenAI(**data) # type: ignore + else: + openai_client = client + + return openai_client + + +class AzureOpenAIFilesAPI(BaseLLM): + """ + AzureOpenAI methods to support for batches + - create_file() + - retrieve_file() + - list_files() + - delete_file() + - file_content() + - update_file() + """ + + def __init__(self) -> None: + super().__init__() + + async def acreate_file( + self, + create_file_data: CreateFileRequest, + openai_client: AsyncAzureOpenAI, + ) -> FileObject: + verbose_logger.debug("create_file_data=%s", create_file_data) + response = await openai_client.files.create(**create_file_data) + verbose_logger.debug("create_file_response=%s", response) + return response + + def create_file( + self, + _is_async: bool, + create_file_data: CreateFileRequest, + api_base: str, + api_key: Optional[str], + api_version: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + ) -> Union[FileObject, Coroutine[Any, Any, FileObject]]: + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + api_version=api_version, + timeout=timeout, + max_retries=max_retries, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.acreate_file( # type: ignore + create_file_data=create_file_data, openai_client=openai_client + ) + response = openai_client.files.create(**create_file_data) + return response + + async def afile_content( + self, + file_content_request: FileContentRequest, + openai_client: AsyncAzureOpenAI, + ) -> HttpxBinaryResponseContent: + response = await openai_client.files.content(**file_content_request) + return response + + def file_content( + self, + _is_async: bool, + file_content_request: FileContentRequest, + api_base: str, + api_key: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + api_version: Optional[str] = None, + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + ) -> Union[ + HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent] + ]: + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + api_version=api_version, + max_retries=max_retries, + organization=organization, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.afile_content( # type: ignore + file_content_request=file_content_request, + openai_client=openai_client, + ) + response = openai_client.files.content(**file_content_request) + + return response + + async def aretrieve_file( + self, + file_id: str, + openai_client: AsyncAzureOpenAI, + ) -> FileObject: + response = await openai_client.files.retrieve(file_id=file_id) + return response + + def retrieve_file( + self, + _is_async: bool, + file_id: str, + api_base: str, + api_key: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + api_version: Optional[str] = None, + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + ): + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + api_version=api_version, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.aretrieve_file( # type: ignore + file_id=file_id, + openai_client=openai_client, + ) + response = openai_client.files.retrieve(file_id=file_id) + + return response + + async def adelete_file( + self, + file_id: str, + openai_client: AsyncAzureOpenAI, + ) -> FileDeleted: + response = await openai_client.files.delete(file_id=file_id) + return response + + def delete_file( + self, + _is_async: bool, + file_id: str, + api_base: str, + api_key: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + api_version: Optional[str] = None, + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + ): + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + api_version=api_version, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.adelete_file( # type: ignore + file_id=file_id, + openai_client=openai_client, + ) + response = openai_client.files.delete(file_id=file_id) + + return response + + async def alist_files( + self, + openai_client: AsyncAzureOpenAI, + purpose: Optional[str] = None, + ): + if isinstance(purpose, str): + response = await openai_client.files.list(purpose=purpose) + else: + response = await openai_client.files.list() + return response + + def list_files( + self, + _is_async: bool, + api_base: str, + api_key: Optional[str], + timeout: Union[float, httpx.Timeout], + max_retries: Optional[int], + organization: Optional[str], + purpose: Optional[str] = None, + api_version: Optional[str] = None, + client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + ): + openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( + get_azure_openai_client( + api_key=api_key, + api_base=api_base, + timeout=timeout, + max_retries=max_retries, + organization=organization, + api_version=api_version, + client=client, + _is_async=_is_async, + ) + ) + if openai_client is None: + raise ValueError( + "AzureOpenAI client is not initialized. Make sure api_key is passed or OPENAI_API_KEY is set in the environment." + ) + + if _is_async is True: + if not isinstance(openai_client, AsyncAzureOpenAI): + raise ValueError( + "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client." + ) + return self.alist_files( # type: ignore + purpose=purpose, + openai_client=openai_client, + ) + + if isinstance(purpose, str): + response = openai_client.files.list(purpose=purpose) + else: + response = openai_client.files.list() + + return response From c6bff3286c2deff1fb31b1fe171b93cc1d235ac0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 15:46:56 -0700 Subject: [PATCH 06/31] test - fine tuning apis --- litellm/tests/test_fine_tuning_api.py | 56 +++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index b7e3c957cce..d35cd661ab8 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -12,6 +12,7 @@ from openai import APITimeoutError as Timeout import litellm litellm.num_retries = 0 +import asyncio import logging from litellm import create_fine_tuning_job @@ -112,3 +113,58 @@ async def test_create_fine_tune_jobs_async(): assert response.status == "cancelled" assert response.id == create_fine_tuning_response.id pass + + +@pytest.mark.asyncio +async def test_azure_create_fine_tune_jobs_async(): + verbose_logger.setLevel(logging.DEBUG) + file_name = "azure_fine_tune.jsonl" + _current_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(_current_dir, file_name) + + file_obj = await litellm.acreate_file( + file=open(file_path, "rb"), + purpose="fine-tune", + custom_llm_provider="azure", + api_key=os.getenv("AZURE_SWEDEN_API_KEY"), + api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", + ) + print("Response from creating file=", file_obj) + + await asyncio.sleep(5) + + create_fine_tuning_response = await litellm.acreate_fine_tuning_job( + model="gpt-35-turbo-1106", + training_file=file_obj.id, + custom_llm_provider="azure", + api_key=os.getenv("AZURE_SWEDEN_API_KEY"), + api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", + ) + + print("response from litellm.create_fine_tuning_job=", create_fine_tuning_response) + + assert create_fine_tuning_response.id is not None + assert create_fine_tuning_response.model == "gpt-35-turbo-1106" + + # # list fine tuning jobs + # print("listing ft jobs") + # ft_jobs = await litellm.alist_fine_tuning_jobs(limit=2) + # print("response from litellm.list_fine_tuning_jobs=", ft_jobs) + # assert len(list(ft_jobs)) > 0 + + # # delete file + + # await litellm.afile_delete( + # file_id=file_obj.id, + # ) + + # # cancel ft job + # response = await litellm.acancel_fine_tuning_job( + # fine_tuning_job_id=create_fine_tuning_response.id, + # ) + + # print("response from litellm.cancel_fine_tuning_job=", response) + + # assert response.status == "cancelled" + # assert response.id == create_fine_tuning_response.id + # pass From 02736ac8b5d776a58c4552b9549f8dc9ff471ebc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 16:03:31 -0700 Subject: [PATCH 07/31] feat FT cancel and LIST endpoints for Azure --- litellm/fine_tuning/main.py | 145 ++++++++++++++++++------- litellm/llms/fine_tuning_apis/azure.py | 9 +- litellm/tests/test_fine_tuning_api.py | 32 +++--- 3 files changed, 132 insertions(+), 54 deletions(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 72119185f22..5206cb78972 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -279,6 +279,25 @@ def cancel_fine_tuning_job( """ try: optional_params = GenericLiteLLMParams(**kwargs) + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _is_async = kwargs.pop("acancel_fine_tuning_job", False) is True + + # OpenAI if custom_llm_provider == "openai": # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -301,25 +320,6 @@ def cancel_fine_tuning_job( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) - ### TIMEOUT LOGIC ### - timeout = ( - optional_params.timeout or kwargs.get("request_timeout", 600) or 600 - ) - # set timeout for 10 minutes by default - - if ( - timeout is not None - and isinstance(timeout, httpx.Timeout) - and supports_httpx_timeout(custom_llm_provider) == False - ): - read_timeout = timeout.read or 600 - timeout = read_timeout # default 10 min timeout - elif timeout is not None and not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore - elif timeout is None: - timeout = 600.0 - - _is_async = kwargs.pop("acancel_fine_tuning_job", False) is True response = openai_fine_tuning_apis_instance.cancel_fine_tuning_job( api_base=api_base, @@ -330,6 +330,40 @@ def cancel_fine_tuning_job( max_retries=optional_params.max_retries, _is_async=_is_async, ) + # Azure OpenAI + elif custom_llm_provider == "azure": + api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token: Optional[str] = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_fine_tuning_apis_instance.cancel_fine_tuning_job( + api_base=api_base, + api_key=api_key, + api_version=api_version, + fine_tuning_job_id=fine_tuning_job_id, + timeout=timeout, + max_retries=optional_params.max_retries, + _is_async=_is_async, + ) else: raise litellm.exceptions.BadRequestError( message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( @@ -405,6 +439,25 @@ def list_fine_tuning_jobs( """ try: optional_params = GenericLiteLLMParams(**kwargs) + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + + _is_async = kwargs.pop("alist_fine_tuning_jobs", False) is True + + # OpenAI if custom_llm_provider == "openai": # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there @@ -427,25 +480,6 @@ def list_fine_tuning_jobs( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) - ### TIMEOUT LOGIC ### - timeout = ( - optional_params.timeout or kwargs.get("request_timeout", 600) or 600 - ) - # set timeout for 10 minutes by default - - if ( - timeout is not None - and isinstance(timeout, httpx.Timeout) - and supports_httpx_timeout(custom_llm_provider) == False - ): - read_timeout = timeout.read or 600 - timeout = read_timeout # default 10 min timeout - elif timeout is not None and not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore - elif timeout is None: - timeout = 600.0 - - _is_async = kwargs.pop("alist_fine_tuning_jobs", False) is True response = openai_fine_tuning_apis_instance.list_fine_tuning_jobs( api_base=api_base, @@ -457,6 +491,41 @@ def list_fine_tuning_jobs( max_retries=optional_params.max_retries, _is_async=_is_async, ) + # Azure OpenAI + elif custom_llm_provider == "azure": + api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore + + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token: Optional[str] = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_fine_tuning_apis_instance.list_fine_tuning_jobs( + api_base=api_base, + api_key=api_key, + api_version=api_version, + after=after, + limit=limit, + timeout=timeout, + max_retries=optional_params.max_retries, + _is_async=_is_async, + ) else: raise litellm.exceptions.BadRequestError( message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( diff --git a/litellm/llms/fine_tuning_apis/azure.py b/litellm/llms/fine_tuning_apis/azure.py index 6c32e2ac788..0e6e0e66d31 100644 --- a/litellm/llms/fine_tuning_apis/azure.py +++ b/litellm/llms/fine_tuning_apis/azure.py @@ -91,13 +91,15 @@ class AzureOpenAIFineTuningAPI(BaseLLM): api_base: Optional[str], timeout: Union[float, httpx.Timeout], max_retries: Optional[int], - organization: Optional[str], + organization: Optional[str] = None, + api_version: Optional[str] = None, client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, ): openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = ( get_azure_openai_client( api_key=api_key, api_base=api_base, + api_version=api_version, timeout=timeout, max_retries=max_retries, organization=organization, @@ -141,8 +143,9 @@ class AzureOpenAIFineTuningAPI(BaseLLM): api_base: Optional[str], timeout: Union[float, httpx.Timeout], max_retries: Optional[int], - organization: Optional[str], + organization: Optional[str] = None, client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, + api_version: Optional[str] = None, after: Optional[str] = None, limit: Optional[int] = None, ): @@ -150,6 +153,7 @@ class AzureOpenAIFineTuningAPI(BaseLLM): get_azure_openai_client( api_key=api_key, api_base=api_base, + api_version=api_version, timeout=timeout, max_retries=max_retries, organization=organization, @@ -175,4 +179,3 @@ class AzureOpenAIFineTuningAPI(BaseLLM): verbose_logger.debug("list fine tuning job, after= %s, limit= %s", after, limit) response = openai_client.fine_tuning.jobs.list(after=after, limit=limit) # type: ignore return response - pass diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index d35cd661ab8..1f99d63582a 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -146,11 +146,15 @@ async def test_azure_create_fine_tune_jobs_async(): assert create_fine_tuning_response.id is not None assert create_fine_tuning_response.model == "gpt-35-turbo-1106" - # # list fine tuning jobs - # print("listing ft jobs") - # ft_jobs = await litellm.alist_fine_tuning_jobs(limit=2) - # print("response from litellm.list_fine_tuning_jobs=", ft_jobs) - # assert len(list(ft_jobs)) > 0 + # list fine tuning jobs + print("listing ft jobs") + ft_jobs = await litellm.alist_fine_tuning_jobs( + limit=2, + custom_llm_provider="azure", + api_key=os.getenv("AZURE_SWEDEN_API_KEY"), + api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", + ) + print("response from litellm.list_fine_tuning_jobs=", ft_jobs) # # delete file @@ -158,13 +162,15 @@ async def test_azure_create_fine_tune_jobs_async(): # file_id=file_obj.id, # ) - # # cancel ft job - # response = await litellm.acancel_fine_tuning_job( - # fine_tuning_job_id=create_fine_tuning_response.id, - # ) + # cancel ft job + response = await litellm.acancel_fine_tuning_job( + fine_tuning_job_id=create_fine_tuning_response.id, + custom_llm_provider="azure", + api_key=os.getenv("AZURE_SWEDEN_API_KEY"), + api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", + ) - # print("response from litellm.cancel_fine_tuning_job=", response) + print("response from litellm.cancel_fine_tuning_job=", response) - # assert response.status == "cancelled" - # assert response.id == create_fine_tuning_response.id - # pass + assert response.status == "cancelled" + assert response.id == create_fine_tuning_response.id From 9b923e66e9f1ece464a7884487b701b0c6053fc2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 16:46:06 -0700 Subject: [PATCH 08/31] test azure fine tune job create --- litellm/files/main.py | 70 +++++++++++++++++++-------- litellm/llms/files_apis/azure.py | 2 +- litellm/tests/test_fine_tuning_api.py | 19 +------- 3 files changed, 52 insertions(+), 39 deletions(-) diff --git a/litellm/files/main.py b/litellm/files/main.py index b3fbd775f61..2440f6d1807 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -198,7 +198,7 @@ async def afile_delete( def file_delete( file_id: str, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -210,6 +210,22 @@ def file_delete( """ try: optional_params = GenericLiteLLMParams(**kwargs) + ### TIMEOUT LOGIC ### + timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 + # set timeout for 10 minutes by default + + if ( + timeout is not None + and isinstance(timeout, httpx.Timeout) + and supports_httpx_timeout(custom_llm_provider) == False + ): + read_timeout = timeout.read or 600 + timeout = read_timeout # default 10 min timeout + elif timeout is not None and not isinstance(timeout, httpx.Timeout): + timeout = float(timeout) # type: ignore + elif timeout is None: + timeout = 600.0 + _is_async = kwargs.pop("is_async", False) is True if custom_llm_provider == "openai": # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there api_base = ( @@ -231,26 +247,6 @@ def file_delete( or litellm.openai_key or os.getenv("OPENAI_API_KEY") ) - ### TIMEOUT LOGIC ### - timeout = ( - optional_params.timeout or kwargs.get("request_timeout", 600) or 600 - ) - # set timeout for 10 minutes by default - - if ( - timeout is not None - and isinstance(timeout, httpx.Timeout) - and supports_httpx_timeout(custom_llm_provider) == False - ): - read_timeout = timeout.read or 600 - timeout = read_timeout # default 10 min timeout - elif timeout is not None and not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore - elif timeout is None: - timeout = 600.0 - - _is_async = kwargs.pop("is_async", False) is True - response = openai_files_instance.delete_file( file_id=file_id, _is_async=_is_async, @@ -260,6 +256,38 @@ def file_delete( max_retries=optional_params.max_retries, organization=organization, ) + elif custom_llm_provider == "azure": + api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE") # type: ignore + api_version = ( + optional_params.api_version + or litellm.api_version + or get_secret("AZURE_API_VERSION") + ) # type: ignore + + api_key = ( + optional_params.api_key + or litellm.api_key + or litellm.azure_key + or get_secret("AZURE_OPENAI_API_KEY") + or get_secret("AZURE_API_KEY") + ) # type: ignore + + extra_body = optional_params.get("extra_body", {}) + azure_ad_token: Optional[str] = None + if extra_body is not None: + azure_ad_token = extra_body.pop("azure_ad_token", None) + else: + azure_ad_token = get_secret("AZURE_AD_TOKEN") # type: ignore + + response = azure_files_instance.delete_file( + _is_async=_is_async, + api_base=api_base, + api_key=api_key, + api_version=api_version, + timeout=timeout, + max_retries=optional_params.max_retries, + file_id=file_id, + ) else: raise litellm.exceptions.BadRequestError( message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format( diff --git a/litellm/llms/files_apis/azure.py b/litellm/llms/files_apis/azure.py index d46d9225a4b..c4c9ee48af8 100644 --- a/litellm/llms/files_apis/azure.py +++ b/litellm/llms/files_apis/azure.py @@ -223,7 +223,7 @@ class AzureOpenAIFilesAPI(BaseLLM): api_key: Optional[str], timeout: Union[float, httpx.Timeout], max_retries: Optional[int], - organization: Optional[str], + organization: Optional[str] = None, api_version: Optional[str] = None, client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None, ): diff --git a/litellm/tests/test_fine_tuning_api.py b/litellm/tests/test_fine_tuning_api.py index 1f99d63582a..f2bb73efde1 100644 --- a/litellm/tests/test_fine_tuning_api.py +++ b/litellm/tests/test_fine_tuning_api.py @@ -122,20 +122,11 @@ async def test_azure_create_fine_tune_jobs_async(): _current_dir = os.path.dirname(os.path.abspath(__file__)) file_path = os.path.join(_current_dir, file_name) - file_obj = await litellm.acreate_file( - file=open(file_path, "rb"), - purpose="fine-tune", - custom_llm_provider="azure", - api_key=os.getenv("AZURE_SWEDEN_API_KEY"), - api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", - ) - print("Response from creating file=", file_obj) - - await asyncio.sleep(5) + file_id = "file-5e4b20ecbd724182b9964f3cd2ab7212" create_fine_tuning_response = await litellm.acreate_fine_tuning_job( model="gpt-35-turbo-1106", - training_file=file_obj.id, + training_file=file_id, custom_llm_provider="azure", api_key=os.getenv("AZURE_SWEDEN_API_KEY"), api_base="https://my-endpoint-sweden-berri992.openai.azure.com/", @@ -156,12 +147,6 @@ async def test_azure_create_fine_tune_jobs_async(): ) print("response from litellm.list_fine_tuning_jobs=", ft_jobs) - # # delete file - - # await litellm.afile_delete( - # file_id=file_obj.id, - # ) - # cancel ft job response = await litellm.acancel_fine_tuning_job( fine_tuning_job_id=create_fine_tuning_response.id, From cdaaf15c8cdce1950841ee51dfbdc9c69f610372 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 30 Jul 2024 16:55:17 -0700 Subject: [PATCH 09/31] fix linting checks --- litellm/files/main.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/litellm/files/main.py b/litellm/files/main.py index 2440f6d1807..49d35539899 100644 --- a/litellm/files/main.py +++ b/litellm/files/main.py @@ -35,7 +35,7 @@ azure_files_instance = AzureOpenAIFilesAPI() async def afile_retrieve( file_id: str, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -75,7 +75,7 @@ async def afile_retrieve( def file_retrieve( file_id: str, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -158,7 +158,7 @@ def file_retrieve( # Delete file async def afile_delete( file_id: str, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -308,7 +308,7 @@ def file_delete( # List files async def afile_list( - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", purpose: Optional[str] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -348,7 +348,7 @@ async def afile_list( def file_list( - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", purpose: Optional[str] = None, extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, @@ -474,7 +474,7 @@ async def acreate_file( def create_file( file: FileTypes, purpose: Literal["assistants", "batch", "fine-tune"], - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -593,7 +593,7 @@ def create_file( async def afile_content( file_id: str, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, @@ -633,7 +633,7 @@ async def afile_content( def file_content( file_id: str, - custom_llm_provider: Literal["openai"] = "openai", + custom_llm_provider: Literal["openai", "azure"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, From 415611b7f2a6014b200123ec7118ff09cd6a33e5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 11:57:52 -0700 Subject: [PATCH 10/31] feat add POST /v1/fine_tuning/jobs --- .../fine_tuning_endpoints.py | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py diff --git a/litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py b/litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py new file mode 100644 index 00000000000..36531a82607 --- /dev/null +++ b/litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py @@ -0,0 +1,157 @@ +######################################################################### + +# /v1/fine_tuning Endpoints + +# Equivalent of https://platform.openai.com/docs/api-reference/fine-tuning +########################################################################## + +import asyncio +import traceback +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import fastapi +import httpx +from fastapi import ( + APIRouter, + Depends, + File, + Form, + Header, + HTTPException, + Request, + Response, + UploadFile, + status, +) + +import litellm +from litellm import CreateFileRequest, FileContentRequest +from litellm._logging import verbose_proxy_logger +from litellm.batches.main import FileObject +from litellm.proxy._types import * +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() + +from litellm.llms.fine_tuning_apis.openai import ( + FineTuningJob, + FineTuningJobCreate, + OpenAIFineTuningAPI, +) + + +@router.post( + "/v1/fine_tuning/jobs", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) +@router.post( + "/fine_tuning/jobs", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) +async def create_fine_tuning_job( + request: Request, + fastapi_response: Response, + fine_tuning_job: FineTuningJobCreate, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + This is the equivalent of POST https://api.openai.com/v1/fine_tuning/jobs + + Supports Identical Params as: https://platform.openai.com/docs/api-reference/fine-tuning/create + + Example Curl: + ``` + curl http://localhost:4000/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-3.5-turbo", + "training_file": "file-abc123", + "hyperparameters": { + "n_epochs": 4 + } + }' + ``` + """ + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + get_custom_headers, + proxy_config, + proxy_logging_obj, + version, + ) + + try: + # Convert Pydantic model to dict + data = fine_tuning_job.dict(exclude_unset=True) + + # Include original request and headers in the data + data = await add_litellm_data_to_request( + data=data, + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + # For now, use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for fine-tuning + response = await litellm.acreate_fine_tuning_job( + custom_llm_provider="openai", **data + ) + + ### ALERTING ### + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=data.get("litellm_call_id", ""), status="success" + ) + ) + + ### RESPONSE HEADERS ### + hidden_params = getattr(response, "_hidden_params", {}) or {} + model_id = hidden_params.get("model_id", None) or "" + cache_key = hidden_params.get("cache_key", None) or "" + api_base = hidden_params.get("api_base", None) or "" + + fastapi_response.headers.update( + get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + ) + ) + + return response + except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data + ) + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e.detail)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) From ef5aeb17a1c30987bfa099adbb0cd0f0032d07e4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 12:41:39 -0700 Subject: [PATCH 11/31] fix pydantic obj for FT endpoints --- litellm/fine_tuning/main.py | 10 +- litellm/llms/fine_tuning_apis/openai.py | 8 +- .../fine_tuning_endpoints.py | 157 ------------------ litellm/types/llms/openai.py | 45 ++--- 4 files changed, 36 insertions(+), 184 deletions(-) delete mode 100644 litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 5206cb78972..ede85351d11 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -18,6 +18,7 @@ import httpx import litellm from litellm import get_secret +from litellm._logging import verbose_logger from litellm.llms.fine_tuning_apis.azure import AzureOpenAIFineTuningAPI from litellm.llms.fine_tuning_apis.openai import ( FineTuningJob, @@ -51,6 +52,9 @@ async def acreate_fine_tuning_job( Async: Creates and executes a batch from an uploaded file of request """ + verbose_logger.debug( + "inside acreate_fine_tuning_job model=%s and kwargs=%s", model, kwargs + ) try: loop = asyncio.get_event_loop() kwargs["acreate_fine_tuning_job"] = True @@ -156,11 +160,15 @@ def create_fine_tuning_job( seed=seed, ) + create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump( + exclude_none=True + ) + response = openai_fine_tuning_apis_instance.create_fine_tuning_job( api_base=api_base, api_key=api_key, organization=organization, - create_fine_tuning_job_data=create_fine_tuning_job_data, + create_fine_tuning_job_data=create_fine_tuning_job_data_dict, timeout=timeout, max_retries=optional_params.max_retries, _is_async=_is_async, diff --git a/litellm/llms/fine_tuning_apis/openai.py b/litellm/llms/fine_tuning_apis/openai.py index 2f6d89ea0b3..6f3cd60211a 100644 --- a/litellm/llms/fine_tuning_apis/openai.py +++ b/litellm/llms/fine_tuning_apis/openai.py @@ -50,18 +50,18 @@ class OpenAIFineTuningAPI(BaseLLM): async def acreate_fine_tuning_job( self, - create_fine_tuning_job_data: FineTuningJobCreate, + create_fine_tuning_job_data: dict, openai_client: AsyncOpenAI, ) -> FineTuningJob: response = await openai_client.fine_tuning.jobs.create( - **create_fine_tuning_job_data # type: ignore + **create_fine_tuning_job_data ) return response def create_fine_tuning_job( self, _is_async: bool, - create_fine_tuning_job_data: FineTuningJobCreate, + create_fine_tuning_job_data: dict, api_key: Optional[str], api_base: Optional[str], timeout: Union[float, httpx.Timeout], @@ -95,7 +95,7 @@ class OpenAIFineTuningAPI(BaseLLM): verbose_logger.debug( "creating fine tuning job, args= %s", create_fine_tuning_job_data ) - response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) # type: ignore + response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) return response async def acancel_fine_tuning_job( diff --git a/litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py b/litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py deleted file mode 100644 index 36531a82607..00000000000 --- a/litellm/proxy/fine_tuning_endpoints.py/fine_tuning_endpoints.py +++ /dev/null @@ -1,157 +0,0 @@ -######################################################################### - -# /v1/fine_tuning Endpoints - -# Equivalent of https://platform.openai.com/docs/api-reference/fine-tuning -########################################################################## - -import asyncio -import traceback -from datetime import datetime, timedelta, timezone -from typing import List, Optional - -import fastapi -import httpx -from fastapi import ( - APIRouter, - Depends, - File, - Form, - Header, - HTTPException, - Request, - Response, - UploadFile, - status, -) - -import litellm -from litellm import CreateFileRequest, FileContentRequest -from litellm._logging import verbose_proxy_logger -from litellm.batches.main import FileObject -from litellm.proxy._types import * -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth - -router = APIRouter() - -from litellm.llms.fine_tuning_apis.openai import ( - FineTuningJob, - FineTuningJobCreate, - OpenAIFineTuningAPI, -) - - -@router.post( - "/v1/fine_tuning/jobs", - dependencies=[Depends(user_api_key_auth)], - tags=["fine-tuning"], -) -@router.post( - "/fine_tuning/jobs", - dependencies=[Depends(user_api_key_auth)], - tags=["fine-tuning"], -) -async def create_fine_tuning_job( - request: Request, - fastapi_response: Response, - fine_tuning_job: FineTuningJobCreate, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - Creates a fine-tuning job which begins the process of creating a new model from a given dataset. - This is the equivalent of POST https://api.openai.com/v1/fine_tuning/jobs - - Supports Identical Params as: https://platform.openai.com/docs/api-reference/fine-tuning/create - - Example Curl: - ``` - curl http://localhost:4000/v1/fine_tuning/jobs \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-1234" \ - -d '{ - "model": "gpt-3.5-turbo", - "training_file": "file-abc123", - "hyperparameters": { - "n_epochs": 4 - } - }' - ``` - """ - from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, - general_settings, - get_custom_headers, - proxy_config, - proxy_logging_obj, - version, - ) - - try: - # Convert Pydantic model to dict - data = fine_tuning_job.dict(exclude_unset=True) - - # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, - request=request, - general_settings=general_settings, - user_api_key_dict=user_api_key_dict, - version=version, - proxy_config=proxy_config, - ) - - # For now, use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for fine-tuning - response = await litellm.acreate_fine_tuning_job( - custom_llm_provider="openai", **data - ) - - ### ALERTING ### - asyncio.create_task( - proxy_logging_obj.update_request_status( - litellm_call_id=data.get("litellm_call_id", ""), status="success" - ) - ) - - ### RESPONSE HEADERS ### - hidden_params = getattr(response, "_hidden_params", {}) or {} - model_id = hidden_params.get("model_id", None) or "" - cache_key = hidden_params.get("cache_key", None) or "" - api_base = hidden_params.get("api_base", None) or "" - - fastapi_response.headers.update( - get_custom_headers( - user_api_key_dict=user_api_key_dict, - model_id=model_id, - cache_key=cache_key, - api_base=api_base, - version=version, - model_region=getattr(user_api_key_dict, "allowed_model_region", ""), - ) - ) - - return response - except Exception as e: - await proxy_logging_obj.post_call_failure_hook( - user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data - ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {}".format( - str(e) - ) - ) - verbose_proxy_logger.debug(traceback.format_exc()) - if isinstance(e, HTTPException): - raise ProxyException( - message=getattr(e, "message", str(e.detail)), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), - ) - else: - error_msg = f"{str(e)}" - raise ProxyException( - message=getattr(e, "message", error_msg), - type=getattr(e, "type", "None"), - param=getattr(e, "param", "None"), - code=getattr(e, "status_code", 500), - ) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 396e58e994f..3bb59f00533 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -9,7 +9,6 @@ from typing import ( Mapping, Optional, Tuple, - TypedDict, Union, ) @@ -31,7 +30,7 @@ from openai.types.beta.threads.message import Message as OpenAIMessage from openai.types.beta.threads.message_content import MessageContent from openai.types.beta.threads.run import Run from pydantic import BaseModel, Field -from typing_extensions import Dict, Required, override +from typing_extensions import Dict, Required, TypedDict, override FileContent = Union[IO[bytes], bytes, PathLike] @@ -457,15 +456,17 @@ class ChatCompletionUsageBlock(TypedDict): total_tokens: int -class Hyperparameters(TypedDict): - batch_size: Optional[Union[str, int]] # "Number of examples in each batch." - learning_rate_multiplier: Optional[ - Union[str, float] - ] # Scaling factor for the learning rate - n_epochs: Optional[Union[str, int]] # "The number of epochs to train the model for" +class Hyperparameters(BaseModel): + batch_size: Optional[Union[str, int]] = None # "Number of examples in each batch." + learning_rate_multiplier: Optional[Union[str, float]] = ( + None # Scaling factor for the learning rate + ) + n_epochs: Optional[Union[str, int]] = ( + None # "The number of epochs to train the model for" + ) -class FineTuningJobCreate(TypedDict): +class FineTuningJobCreate(BaseModel): """ FineTuningJobCreate - Create a fine-tuning job @@ -489,16 +490,16 @@ class FineTuningJobCreate(TypedDict): model: str # "The name of the model to fine-tune." training_file: str # "The ID of an uploaded file that contains training data." - hyperparameters: Optional[ - Hyperparameters - ] # "The hyperparameters used for the fine-tuning job." - suffix: Optional[ - str - ] # "A string of up to 18 characters that will be added to your fine-tuned model name." - validation_file: Optional[ - str - ] # "The ID of an uploaded file that contains validation data." - integrations: Optional[ - List[str] - ] # "A list of integrations to enable for your fine-tuning job." - seed: Optional[int] # "The seed controls the reproducibility of the job." + hyperparameters: Optional[Hyperparameters] = ( + None # "The hyperparameters used for the fine-tuning job." + ) + suffix: Optional[str] = ( + None # "A string of up to 18 characters that will be added to your fine-tuned model name." + ) + validation_file: Optional[str] = ( + None # "The ID of an uploaded file that contains validation data." + ) + integrations: Optional[List[str]] = ( + None # "A list of integrations to enable for your fine-tuning job." + ) + seed: Optional[int] = None # "The seed controls the reproducibility of the job." From 9d90f174a7c8cf199d9d70e3ec077f137bed9e8d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 12:44:01 -0700 Subject: [PATCH 12/31] fix endpoint to create fine tuning jobs --- .../proxy/fine_tuning_endpoints/endpoints.py | 160 ++++++++++++++++++ litellm/proxy/proxy_server.py | 2 + 2 files changed, 162 insertions(+) create mode 100644 litellm/proxy/fine_tuning_endpoints/endpoints.py diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py new file mode 100644 index 00000000000..b15de075fa8 --- /dev/null +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -0,0 +1,160 @@ +######################################################################### + +# /v1/fine_tuning Endpoints + +# Equivalent of https://platform.openai.com/docs/api-reference/fine-tuning +########################################################################## + +import asyncio +import traceback +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +import fastapi +import httpx +from fastapi import ( + APIRouter, + Depends, + File, + Form, + Header, + HTTPException, + Request, + Response, + UploadFile, + status, +) + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.batches.main import FileObject +from litellm.proxy._types import * +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + +router = APIRouter() + +from litellm.llms.fine_tuning_apis.openai import ( + FineTuningJob, + FineTuningJobCreate, + OpenAIFineTuningAPI, +) + + +@router.post( + "/v1/fine_tuning/jobs", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) +@router.post( + "/fine_tuning/jobs", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) +async def create_fine_tuning_job( + request: Request, + fastapi_response: Response, + fine_tuning_request: FineTuningJobCreate, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Creates a fine-tuning job which begins the process of creating a new model from a given dataset. + This is the equivalent of POST https://api.openai.com/v1/fine_tuning/jobs + + Supports Identical Params as: https://platform.openai.com/docs/api-reference/fine-tuning/create + + Example Curl: + ``` + curl http://localhost:4000/v1/fine_tuning/jobs \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "gpt-3.5-turbo", + "training_file": "file-abc123", + "hyperparameters": { + "n_epochs": 4 + } + }' + ``` + """ + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + get_custom_headers, + proxy_config, + proxy_logging_obj, + version, + ) + + try: + # Convert Pydantic model to dict + data = fine_tuning_request.model_dump(exclude_none=True) + + verbose_proxy_logger.debug( + "Request received by LiteLLM:\n{}".format(json.dumps(data, indent=4)), + ) + + # Include original request and headers in the data + data = await add_litellm_data_to_request( + data=data, + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + # For now, use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for fine-tuning + response = await litellm.acreate_fine_tuning_job( + custom_llm_provider="openai", **data + ) + + ### ALERTING ### + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=data.get("litellm_call_id", ""), status="success" + ) + ) + + ### RESPONSE HEADERS ### + hidden_params = getattr(response, "_hidden_params", {}) or {} + model_id = hidden_params.get("model_id", None) or "" + cache_key = hidden_params.get("cache_key", None) or "" + api_base = hidden_params.get("api_base", None) or "" + + fastapi_response.headers.update( + get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + ) + ) + + return response + except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data + ) + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e.detail)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 5a2970df51d..a995980f276 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -153,6 +153,7 @@ from litellm.proxy.common_utils.init_callbacks import initialize_callbacks_on_pr from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) +from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router from litellm.proxy.guardrails.init_guardrails import initialize_guardrails from litellm.proxy.health_check import perform_health_check from litellm.proxy.health_endpoints._health_endpoints import router as health_router @@ -9608,3 +9609,4 @@ app.include_router(debugging_endpoints_router) app.include_router(ui_crud_endpoints_router) app.include_router(openai_files_router) app.include_router(team_callback_router) +app.include_router(fine_tuning_router) From bd7b485d09a6097fda745b5f0bd436ab12a5a8cc Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 13:31:00 -0700 Subject: [PATCH 13/31] read ft config --- litellm/proxy/proxy_config.yaml | 21 +++++++++++++++++++++ litellm/proxy/proxy_server.py | 8 ++++++++ 2 files changed, 29 insertions(+) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index f7e5a894f01..dec13898e38 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -25,6 +25,27 @@ model_list: api_key: "os.environ/OPENAI_API_KEY" model_info: mode: audio_speech + +# For /fine_tuning/jobs endpoints +finetune_settings: + - custom_llm_provider: azure + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: fake-key + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: fake-key + +# for /files endpoints +files_settings: + - custom_llm_provider: azure + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: fake-key + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: fake-key + + + general_settings: master_key: sk-1234 diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a995980f276..9f96dd286dd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -154,6 +154,7 @@ from litellm.proxy.common_utils.openai_endpoint_utils import ( remove_sensitive_info_from_deployment, ) from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router +from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config from litellm.proxy.guardrails.init_guardrails import initialize_guardrails from litellm.proxy.health_check import perform_health_check from litellm.proxy.health_endpoints._health_endpoints import router as health_router @@ -1808,6 +1809,13 @@ class ProxyConfig: assistant_settings["litellm_params"][k] = v assistants_config = AssistantsTypedDict(**assistant_settings) # type: ignore + ## /fine_tuning/jobs endpoints config + finetuning_config = config.get("finetune_settings", None) + set_fine_tuning_config(config=finetuning_config) + + ## /files endpoint config + files_config = config.get("files_settings", None) + ## ROUTER SETTINGS (e.g. routing_strategy, ...) router_settings = config.get("router_settings", None) if router_settings and isinstance(router_settings, dict): From e4c73036fc1e785b384fdeb9439afdc4d063e9fe Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 13:32:18 -0700 Subject: [PATCH 14/31] validation for passing config file --- .../proxy/fine_tuning_endpoints/endpoints.py | 42 +++++++++++++++---- litellm/types/llms/openai.py | 4 ++ 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index b15de075fa8..9c58337d1d8 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -33,11 +33,29 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router = APIRouter() -from litellm.llms.fine_tuning_apis.openai import ( - FineTuningJob, - FineTuningJobCreate, - OpenAIFineTuningAPI, -) +from litellm.types.llms.openai import LiteLLMFineTuningJobCreate + +fine_tuning_config = None + + +def set_fine_tuning_config(config): + global fine_tuning_config + fine_tuning_config = config + + +# Function to search for specific custom_llm_provider and return its configuration +def get_provider_config( + custom_llm_provider: str, +): + global fine_tuning_config + if fine_tuning_config is None: + raise ValueError( + "fine_tuning_config is not set, set it on your config.yaml file." + ) + for setting in fine_tuning_config: + if setting.get("custom_llm_provider") == custom_llm_provider: + return setting + return None @router.post( @@ -53,7 +71,7 @@ from litellm.llms.fine_tuning_apis.openai import ( async def create_fine_tuning_job( request: Request, fastapi_response: Response, - fine_tuning_request: FineTuningJobCreate, + fine_tuning_request: LiteLLMFineTuningJobCreate, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -103,11 +121,17 @@ async def create_fine_tuning_job( proxy_config=proxy_config, ) - # For now, use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for fine-tuning - response = await litellm.acreate_fine_tuning_job( - custom_llm_provider="openai", **data + # get configs for custom_llm_provider + llm_provider_config = get_provider_config( + custom_llm_provider=fine_tuning_request.custom_llm_provider, ) + # add llm_provider_config to data + data.update(llm_provider_config) + + # For now, use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for fine-tuning + response = await litellm.acreate_fine_tuning_job(**data) + ### ALERTING ### asyncio.create_task( proxy_logging_obj.update_request_status( diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 3bb59f00533..875ccadf14e 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -503,3 +503,7 @@ class FineTuningJobCreate(BaseModel): None # "A list of integrations to enable for your fine-tuning job." ) seed: Optional[int] = None # "The seed controls the reproducibility of the job." + + +class LiteLLMFineTuningJobCreate(FineTuningJobCreate): + custom_llm_provider: Literal["openai", "azure"] From 2cf10a621e09f37a0db442809d8bda5eb4736fd7 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 13:50:51 -0700 Subject: [PATCH 15/31] add /fine_tuning/jobs routes --- litellm/proxy/_types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 8d9eb636d3b..53fed823e54 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -208,6 +208,8 @@ class LiteLLMRoutes(enum.Enum): "/files/{file_id}", "/v1/files/{file_id}/content", "/files/{file_id}/content", + # fine_tuning + "/fine_tuning/jobs", # assistants-related routes "/assistants", "/v1/assistants", From ff9bb96217489d84d5b9150a6ab3f6b61231e3c9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 13:57:58 -0700 Subject: [PATCH 16/31] feat support azure ft create endpoint --- litellm/fine_tuning/main.py | 6 +++++- litellm/llms/fine_tuning_apis/azure.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index ede85351d11..1400d0b1ad8 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -208,11 +208,15 @@ def create_fine_tuning_job( seed=seed, ) + create_fine_tuning_job_data_dict = create_fine_tuning_job_data.model_dump( + exclude_none=True + ) + response = azure_fine_tuning_apis_instance.create_fine_tuning_job( api_base=api_base, api_key=api_key, api_version=api_version, - create_fine_tuning_job_data=create_fine_tuning_job_data, + create_fine_tuning_job_data=create_fine_tuning_job_data_dict, timeout=timeout, max_retries=optional_params.max_retries, _is_async=_is_async, diff --git a/litellm/llms/fine_tuning_apis/azure.py b/litellm/llms/fine_tuning_apis/azure.py index 0e6e0e66d31..ff7d40ff8c1 100644 --- a/litellm/llms/fine_tuning_apis/azure.py +++ b/litellm/llms/fine_tuning_apis/azure.py @@ -21,7 +21,7 @@ class AzureOpenAIFineTuningAPI(BaseLLM): async def acreate_fine_tuning_job( self, - create_fine_tuning_job_data: FineTuningJobCreate, + create_fine_tuning_job_data: dict, openai_client: AsyncAzureOpenAI, ) -> FineTuningJob: response = await openai_client.fine_tuning.jobs.create( @@ -32,7 +32,7 @@ class AzureOpenAIFineTuningAPI(BaseLLM): def create_fine_tuning_job( self, _is_async: bool, - create_fine_tuning_job_data: FineTuningJobCreate, + create_fine_tuning_job_data: dict, api_key: Optional[str], api_base: Optional[str], timeout: Union[float, httpx.Timeout], From 2371df9deddc2f617e3a9b48b413dbfeba12473d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 15:28:41 -0700 Subject: [PATCH 17/31] allow setting files config --- litellm/proxy/proxy_server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9f96dd286dd..1e44c44188e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -181,6 +181,7 @@ from litellm.proxy.management_endpoints.team_endpoints import router as team_rou from litellm.proxy.openai_files_endpoints.files_endpoints import ( router as openai_files_router, ) +from litellm.proxy.openai_files_endpoints.files_endpoints import set_files_config from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( initialize_pass_through_endpoints, ) @@ -1815,6 +1816,7 @@ class ProxyConfig: ## /files endpoint config files_config = config.get("files_settings", None) + set_files_config(config=files_config) ## ROUTER SETTINGS (e.g. routing_strategy, ...) router_settings = config.get("router_settings", None) From c8dfc95e90773666075fb5f74431920e37204bf0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 15:29:06 -0700 Subject: [PATCH 18/31] add examples on config --- litellm/proxy/proxy_config.yaml | 8 ++++---- proxy_server_config.yaml | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index dec13898e38..20c08fe63c1 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -29,20 +29,20 @@ model_list: # For /fine_tuning/jobs endpoints finetune_settings: - custom_llm_provider: azure - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_base: https://exampleopenaiendpoint-production.up.railway.app api_key: fake-key api_version: "2023-03-15-preview" - custom_llm_provider: openai - api_key: fake-key + api_key: os.environ/OPENAI_API_KEY # for /files endpoints files_settings: - custom_llm_provider: azure - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_base: http://0.0.0.0:8090 api_key: fake-key api_version: "2023-03-15-preview" - custom_llm_provider: openai - api_key: fake-key + api_key: os.environ/OPENAI_API_KEY diff --git a/proxy_server_config.yaml b/proxy_server_config.yaml index f7766b65bfe..67e936a138b 100644 --- a/proxy_server_config.yaml +++ b/proxy_server_config.yaml @@ -120,6 +120,24 @@ litellm_settings: langfuse_secret: os.environ/LANGFUSE_PROJECT2_SECRET # Project 2 langfuse_host: https://us.cloud.langfuse.com +# For /fine_tuning/jobs endpoints +finetune_settings: + - custom_llm_provider: azure + api_base: https://exampleopenaiendpoint-production.up.railway.app + api_key: fake-key + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + +# for /files endpoints +files_settings: + - custom_llm_provider: azure + api_base: http://0.0.0.0:8090 + api_key: fake-key + api_version: "2023-03-15-preview" + - custom_llm_provider: openai + api_key: os.environ/OPENAI_API_KEY + router_settings: routing_strategy: usage-based-routing-v2 redis_host: os.environ/REDIS_HOST From 287b09cff6e8f6ce3aef3174bc2b042890d6e63a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 15:30:26 -0700 Subject: [PATCH 19/31] add test for ft endpoints on azure --- .../proxy/fine_tuning_endpoints/endpoints.py | 13 +++++- .../openai_files_endpoints/files_endpoints.py | 41 +++++++++++++++++-- tests/openai_batch_completions.jsonl | 2 + tests/test_openai_fine_tuning.py | 23 +++++++++++ 4 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 tests/openai_batch_completions.jsonl create mode 100644 tests/test_openai_fine_tuning.py diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 9c58337d1d8..c560e7b535f 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -40,11 +40,20 @@ fine_tuning_config = None def set_fine_tuning_config(config): global fine_tuning_config + if not isinstance(config, list): + raise ValueError("invalid fine_tuning config, expected a list is not a list") + + for element in config: + if isinstance(element, dict): + for key, value in element.items(): + if isinstance(value, str) and value.startswith("os.environ/"): + element[key] = litellm.get_secret(value) + fine_tuning_config = config # Function to search for specific custom_llm_provider and return its configuration -def get_provider_config( +def get_fine_tuning_provider_config( custom_llm_provider: str, ): global fine_tuning_config @@ -122,7 +131,7 @@ async def create_fine_tuning_job( ) # get configs for custom_llm_provider - llm_provider_config = get_provider_config( + llm_provider_config = get_fine_tuning_provider_config( custom_llm_provider=fine_tuning_request.custom_llm_provider, ) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index e4ac8fc8042..325bfa6349c 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -34,6 +34,34 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth router = APIRouter() +files_config = None + + +def set_files_config(config): + global files_config + if not isinstance(config, list): + raise ValueError("invalid files config, expected a list is not a list") + + for element in config: + if isinstance(element, dict): + for key, value in element.items(): + if isinstance(value, str) and value.startswith("os.environ/"): + element[key] = litellm.get_secret(value) + + files_config = config + + +def get_files_provider_config( + custom_llm_provider: str, +): + global files_config + if files_config is None: + raise ValueError("files_config is not set, set it on your config.yaml file.") + for setting in files_config: + if setting.get("custom_llm_provider") == custom_llm_provider: + return setting + return None + @router.post( "/v1/files", @@ -49,6 +77,7 @@ async def create_file( request: Request, fastapi_response: Response, purpose: str = Form(...), + custom_llm_provider: str = Form(...), file: UploadFile = File(...), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): @@ -100,11 +129,17 @@ async def create_file( _create_file_request = CreateFileRequest(file=file_data, **data) - # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch - response = await litellm.acreate_file( - custom_llm_provider="openai", **_create_file_request + # get configs for custom_llm_provider + llm_provider_config = get_files_provider_config( + custom_llm_provider=custom_llm_provider ) + # add llm_provider_config to data + _create_file_request.update(llm_provider_config) + + # for now use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for acreate_batch + response = await litellm.acreate_file(**_create_file_request) + ### ALERTING ### asyncio.create_task( proxy_logging_obj.update_request_status( diff --git a/tests/openai_batch_completions.jsonl b/tests/openai_batch_completions.jsonl new file mode 100644 index 00000000000..05448952a0f --- /dev/null +++ b/tests/openai_batch_completions.jsonl @@ -0,0 +1,2 @@ +{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} +{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo-0125", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 10}} \ No newline at end of file diff --git a/tests/test_openai_fine_tuning.py b/tests/test_openai_fine_tuning.py new file mode 100644 index 00000000000..94ef156b228 --- /dev/null +++ b/tests/test_openai_fine_tuning.py @@ -0,0 +1,23 @@ +from openai import AsyncOpenAI +import os +import pytest + + +@pytest.mark.asyncio +async def test_openai_fine_tuning(): + """ + [PROD Test] Ensures logprobs are returned correctly + """ + client = AsyncOpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") + + file_name = "openai_batch_completions.jsonl" + _current_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(_current_dir, file_name) + + response = await client.files.create( + extra_body={"custom_llm_provider": "azure"}, + file=open(file_path, "rb"), + purpose="fine-tune", + ) + + print("response from files.create: {}".format(response)) From 9b6231810bff35d40e0f8b6522ba3019c681e137 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 15:58:35 -0700 Subject: [PATCH 20/31] add GET fine_tuning/jobs --- .../proxy/fine_tuning_endpoints/endpoints.py | 103 +++++++++++++++++- litellm/proxy/proxy_config.yaml | 2 +- tests/test_openai_fine_tuning.py | 46 ++++++-- 3 files changed, 141 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index c560e7b535f..2f6ad20cfc5 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -138,7 +138,6 @@ async def create_fine_tuning_job( # add llm_provider_config to data data.update(llm_provider_config) - # For now, use custom_llm_provider=="openai" -> this will change as LiteLLM adds more providers for fine-tuning response = await litellm.acreate_fine_tuning_job(**data) ### ALERTING ### @@ -191,3 +190,105 @@ async def create_fine_tuning_job( param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), ) + + +@router.get( + "/v1/fine_tuning/jobs", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) +async def list_fine_tuning_jobs( + request: Request, + fastapi_response: Response, + custom_llm_provider: Literal["openai", "azure"], + after: Optional[str] = None, + limit: Optional[int] = None, + user_api_key_dict: dict = Depends(user_api_key_auth), +): + """ + Lists fine-tuning jobs for the organization. + This is the equivalent of GET https://api.openai.com/v1/fine_tuning/jobs + + Supported Query Params: + - `custom_llm_provider`: Name of the LiteLLM provider + - `after`: Identifier for the last job from the previous pagination request. + - `limit`: Number of fine-tuning jobs to retrieve (default is 20). + """ + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + get_custom_headers, + proxy_config, + proxy_logging_obj, + version, + ) + + data: dict = {} + try: + # Include original request and headers in the data + data = await add_litellm_data_to_request( + data=data, + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) + + data.update(llm_provider_config) + + response = await litellm.alist_fine_tuning_jobs( + **data, + after=after, + limit=limit, + ) + + ### RESPONSE HEADERS ### + hidden_params = getattr(response, "_hidden_params", {}) or {} + model_id = hidden_params.get("model_id", None) or "" + cache_key = hidden_params.get("cache_key", None) or "" + api_base = hidden_params.get("api_base", None) or "" + + fastapi_response.headers.update( + get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + ) + ) + + return response + + except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data + ) + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e.detail)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 20c08fe63c1..b78832a10aa 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -38,7 +38,7 @@ finetune_settings: # for /files endpoints files_settings: - custom_llm_provider: azure - api_base: http://0.0.0.0:8090 + api_base: https://exampleopenaiendpoint-production.up.railway.app api_key: fake-key api_version: "2023-03-15-preview" - custom_llm_provider: openai diff --git a/tests/test_openai_fine_tuning.py b/tests/test_openai_fine_tuning.py index 94ef156b228..096aabea4a6 100644 --- a/tests/test_openai_fine_tuning.py +++ b/tests/test_openai_fine_tuning.py @@ -10,14 +10,44 @@ async def test_openai_fine_tuning(): """ client = AsyncOpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - file_name = "openai_batch_completions.jsonl" - _current_dir = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(_current_dir, file_name) + # file_name = "openai_batch_completions.jsonl" + # _current_dir = os.path.dirname(os.path.abspath(__file__)) + # file_path = os.path.join(_current_dir, file_name) - response = await client.files.create( - extra_body={"custom_llm_provider": "azure"}, - file=open(file_path, "rb"), - purpose="fine-tune", + # response = await client.files.create( + # extra_body={"custom_llm_provider": "azure"}, + # file=open(file_path, "rb"), + # purpose="fine-tune", + # ) + + # print("response from files.create: {}".format(response)) + + # # create fine tuning job + + # ft_job = await client.fine_tuning.jobs.create( + # model="gpt-35-turbo-1106", + # training_file=response.id, + # extra_body={"custom_llm_provider": "azure"}, + # ) + + # print("response from ft job={}".format(ft_job)) + + # # response from example endpoint + # assert ft_job.id == "file-abc123" + + # get fine tuning job + # specific_ft_job = await client.fine_tuning.jobs.retrieve( + # fine_tuning_job_id="123", + # extra_body={"custom_llm_provider": "azure"}, + # ) + + # list all fine tuning jobs + list_ft_jobs = await client.fine_tuning.jobs.list( + extra_query={"custom_llm_provider": "azure"} ) - print("response from files.create: {}".format(response)) + # cancel specific fine tuning job + cancel_ft_job = await client.fine_tuning.jobs.cancel( + fine_tuning_job_id="123", + extra_body={"custom_llm_provider": "azure"}, + ) From 5f1070e47f8d099cac2daf07ad79d161645fef9d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:12:36 -0700 Subject: [PATCH 21/31] add cancel endpoint --- .../proxy/fine_tuning_endpoints/endpoints.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 2f6ad20cfc5..eadb69f7a2d 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -292,3 +292,103 @@ async def list_fine_tuning_jobs( param=getattr(e, "param", "None"), code=getattr(e, "status_code", 500), ) + + +@router.post( + "/v1/fine_tuning/jobs/{fine_tuning_job_id:path}/cancel", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) +async def retrieve_fine_tuning_job( + request: Request, + fastapi_response: Response, + custom_llm_provider: Literal["openai", "azure"], + fine_tuning_job_id: str, + user_api_key_dict: dict = Depends(user_api_key_auth), +): + """ + Cancel a fine-tuning job. + + This is the equivalent of POST https://api.openai.com/v1/fine_tuning/jobs/{fine_tuning_job_id}/cancel + + Supported Query Params: + - `custom_llm_provider`: Name of the LiteLLM provider + - `fine_tuning_job_id`: The ID of the fine-tuning job to cancel. + """ + from litellm.proxy.proxy_server import ( + add_litellm_data_to_request, + general_settings, + get_custom_headers, + proxy_config, + proxy_logging_obj, + version, + ) + + data: dict = {} + try: + # Include original request and headers in the data + data = await add_litellm_data_to_request( + data=data, + request=request, + general_settings=general_settings, + user_api_key_dict=user_api_key_dict, + version=version, + proxy_config=proxy_config, + ) + + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) + + data.update(llm_provider_config) + + response = await litellm.acancel_fine_tuning_job( + **data, + fine_tuning_job_id=fine_tuning_job_id, + ) + + ### RESPONSE HEADERS ### + hidden_params = getattr(response, "_hidden_params", {}) or {} + model_id = hidden_params.get("model_id", None) or "" + cache_key = hidden_params.get("cache_key", None) or "" + api_base = hidden_params.get("api_base", None) or "" + + fastapi_response.headers.update( + get_custom_headers( + user_api_key_dict=user_api_key_dict, + model_id=model_id, + cache_key=cache_key, + api_base=api_base, + version=version, + model_region=getattr(user_api_key_dict, "allowed_model_region", ""), + ) + ) + + return response + + except Exception as e: + await proxy_logging_obj.post_call_failure_hook( + user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data + ) + verbose_proxy_logger.error( + "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( + str(e) + ) + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "message", str(e.detail)), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST), + ) + else: + error_msg = f"{str(e)}" + raise ProxyException( + message=getattr(e, "message", error_msg), + type=getattr(e, "type", "None"), + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", 500), + ) From 729f876f813a40a6e4b6b819a3766991b55e65e4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:12:52 -0700 Subject: [PATCH 22/31] test cancel ft jobs --- tests/test_openai_fine_tuning.py | 44 +++++++++++++++----------------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/tests/test_openai_fine_tuning.py b/tests/test_openai_fine_tuning.py index 096aabea4a6..db41e45438b 100644 --- a/tests/test_openai_fine_tuning.py +++ b/tests/test_openai_fine_tuning.py @@ -10,36 +10,30 @@ async def test_openai_fine_tuning(): """ client = AsyncOpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000") - # file_name = "openai_batch_completions.jsonl" - # _current_dir = os.path.dirname(os.path.abspath(__file__)) - # file_path = os.path.join(_current_dir, file_name) + file_name = "openai_batch_completions.jsonl" + _current_dir = os.path.dirname(os.path.abspath(__file__)) + file_path = os.path.join(_current_dir, file_name) - # response = await client.files.create( - # extra_body={"custom_llm_provider": "azure"}, - # file=open(file_path, "rb"), - # purpose="fine-tune", - # ) + response = await client.files.create( + extra_body={"custom_llm_provider": "azure"}, + file=open(file_path, "rb"), + purpose="fine-tune", + ) - # print("response from files.create: {}".format(response)) + print("response from files.create: {}".format(response)) - # # create fine tuning job + # create fine tuning job - # ft_job = await client.fine_tuning.jobs.create( - # model="gpt-35-turbo-1106", - # training_file=response.id, - # extra_body={"custom_llm_provider": "azure"}, - # ) + ft_job = await client.fine_tuning.jobs.create( + model="gpt-35-turbo-1106", + training_file=response.id, + extra_body={"custom_llm_provider": "azure"}, + ) - # print("response from ft job={}".format(ft_job)) + print("response from ft job={}".format(ft_job)) - # # response from example endpoint - # assert ft_job.id == "file-abc123" - - # get fine tuning job - # specific_ft_job = await client.fine_tuning.jobs.retrieve( - # fine_tuning_job_id="123", - # extra_body={"custom_llm_provider": "azure"}, - # ) + # response from example endpoint + assert ft_job.id == "file-abc123" # list all fine tuning jobs list_ft_jobs = await client.fine_tuning.jobs.list( @@ -51,3 +45,5 @@ async def test_openai_fine_tuning(): fine_tuning_job_id="123", extra_body={"custom_llm_provider": "azure"}, ) + + assert cancel_ft_job.id is not None From 7d06b77fd91c516dfee7b4ba1e8f0fc0893d1a56 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:19:00 -0700 Subject: [PATCH 23/31] add ft jobs in list of allowed routes --- litellm/proxy/_types.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 53fed823e54..82e78e0d769 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -210,6 +210,9 @@ class LiteLLMRoutes(enum.Enum): "/files/{file_id}/content", # fine_tuning "/fine_tuning/jobs", + "v1/fine_tuning/jobs", + "/fine_tuning/jobs/{fine_tuning_job_id}/cancel" + "/v1/fine_tuning/jobs/{fine_tuning_job_id}/cancel" # assistants-related routes "/assistants", "/v1/assistants", From a6e62da9fb401118ae443a75d43754a289262b7b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:19:15 -0700 Subject: [PATCH 24/31] fix cancel ft job route --- litellm/proxy/fine_tuning_endpoints/endpoints.py | 15 ++++++++++++++- tests/test_openai_fine_tuning.py | 6 +++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index eadb69f7a2d..e95c2bae319 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -197,6 +197,11 @@ async def create_fine_tuning_job( dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], ) +@router.get( + "/fine_tuning/jobs", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) async def list_fine_tuning_jobs( request: Request, fastapi_response: Response, @@ -299,10 +304,14 @@ async def list_fine_tuning_jobs( dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], ) +@router.post( + "/fine_tuning/jobs/{fine_tuning_job_id:path}/cancel", + dependencies=[Depends(user_api_key_auth)], + tags=["fine-tuning"], +) async def retrieve_fine_tuning_job( request: Request, fastapi_response: Response, - custom_llm_provider: Literal["openai", "azure"], fine_tuning_job_id: str, user_api_key_dict: dict = Depends(user_api_key_auth), ): @@ -336,6 +345,10 @@ async def retrieve_fine_tuning_job( proxy_config=proxy_config, ) + request_body = await request.json() + + custom_llm_provider = request_body.get("custom_llm_provider", None) + # get configs for custom_llm_provider llm_provider_config = get_fine_tuning_provider_config( custom_llm_provider=custom_llm_provider diff --git a/tests/test_openai_fine_tuning.py b/tests/test_openai_fine_tuning.py index db41e45438b..6d67d414485 100644 --- a/tests/test_openai_fine_tuning.py +++ b/tests/test_openai_fine_tuning.py @@ -33,17 +33,21 @@ async def test_openai_fine_tuning(): print("response from ft job={}".format(ft_job)) # response from example endpoint - assert ft_job.id == "file-abc123" + assert ft_job.id == "ftjob-abc123" # list all fine tuning jobs list_ft_jobs = await client.fine_tuning.jobs.list( extra_query={"custom_llm_provider": "azure"} ) + print("list of ft jobs={}".format(list_ft_jobs)) + # cancel specific fine tuning job cancel_ft_job = await client.fine_tuning.jobs.cancel( fine_tuning_job_id="123", extra_body={"custom_llm_provider": "azure"}, ) + print("response from cancel ft job={}".format(cancel_ft_job)) + assert cancel_ft_job.id is not None From ddc74dbccca63c09ac27f326e82dbeac8f7d0c94 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:23:36 -0700 Subject: [PATCH 25/31] fix reading files/ft config --- litellm/proxy/fine_tuning_endpoints/endpoints.py | 3 +++ litellm/proxy/openai_files_endpoints/files_endpoints.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index e95c2bae319..a8f01cd4894 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -39,6 +39,9 @@ fine_tuning_config = None def set_fine_tuning_config(config): + if config is None: + return + global fine_tuning_config if not isinstance(config, list): raise ValueError("invalid fine_tuning config, expected a list is not a list") diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 325bfa6349c..330798c9dbc 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -39,6 +39,9 @@ files_config = None def set_files_config(config): global files_config + if config is None: + return + if not isinstance(config, list): raise ValueError("invalid files config, expected a list is not a list") From e626f5c7d616bc7c388f8f5de8977ff8fe90968d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:27:59 -0700 Subject: [PATCH 26/31] fix POST files --- litellm/proxy/openai_files_endpoints/files_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 330798c9dbc..807e02a3a51 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -80,7 +80,7 @@ async def create_file( request: Request, fastapi_response: Response, purpose: str = Form(...), - custom_llm_provider: str = Form(...), + custom_llm_provider: str = Form(default="openai"), file: UploadFile = File(...), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): From 02e552e8f85aa887782ef747532bfe70f806984b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:30:08 -0700 Subject: [PATCH 27/31] fix linting errors --- litellm/proxy/fine_tuning_endpoints/endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index a8f01cd4894..5439ea9b5e9 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -211,7 +211,7 @@ async def list_fine_tuning_jobs( custom_llm_provider: Literal["openai", "azure"], after: Optional[str] = None, limit: Optional[int] = None, - user_api_key_dict: dict = Depends(user_api_key_auth), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Lists fine-tuning jobs for the organization. @@ -316,7 +316,7 @@ async def retrieve_fine_tuning_job( request: Request, fastapi_response: Response, fine_tuning_job_id: str, - user_api_key_dict: dict = Depends(user_api_key_auth), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ Cancel a fine-tuning job. From 424a70c3315ea4ecd78c37df49c394624325ab2b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:36:50 -0700 Subject: [PATCH 28/31] fix routes order --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1e44c44188e..983037df370 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9613,10 +9613,10 @@ app.include_router(key_management_router) app.include_router(internal_user_router) app.include_router(team_router) app.include_router(spend_management_router) +app.include_router(fine_tuning_router) app.include_router(caching_router) app.include_router(analytics_router) app.include_router(debugging_endpoints_router) app.include_router(ui_crud_endpoints_router) app.include_router(openai_files_router) app.include_router(team_callback_router) -app.include_router(fine_tuning_router) From bd15d0f4ca28d26b9ede310e0a85fae8c7e73b2d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:41:14 -0700 Subject: [PATCH 29/31] fix mark fine tuning endpoints as enteprrise --- litellm/proxy/fine_tuning_endpoints/endpoints.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 5439ea9b5e9..8c5ceef0183 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -74,11 +74,13 @@ def get_fine_tuning_provider_config( "/v1/fine_tuning/jobs", dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], + summary="✨ (Enterprise) Create Fine-Tuning Job", ) @router.post( "/fine_tuning/jobs", dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], + summary="✨ (Enterprise) Create Fine-Tuning Job", ) async def create_fine_tuning_job( request: Request, @@ -199,11 +201,13 @@ async def create_fine_tuning_job( "/v1/fine_tuning/jobs", dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], + summary="✨ (Enterprise) List Fine-Tuning Jobs", ) @router.get( "/fine_tuning/jobs", dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], + summary="✨ (Enterprise) List Fine-Tuning Jobs", ) async def list_fine_tuning_jobs( request: Request, @@ -306,11 +310,13 @@ async def list_fine_tuning_jobs( "/v1/fine_tuning/jobs/{fine_tuning_job_id:path}/cancel", dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], + summary="✨ (Enterprise) Cancel Fine-Tuning Jobs", ) @router.post( "/fine_tuning/jobs/{fine_tuning_job_id:path}/cancel", dependencies=[Depends(user_api_key_auth)], tags=["fine-tuning"], + summary="✨ (Enterprise) Cancel Fine-Tuning Jobs", ) async def retrieve_fine_tuning_job( request: Request, From 9759fabe8feea7a1155d183ebf0d1c424311aeed Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:43:54 -0700 Subject: [PATCH 30/31] enforce ft endpoints as premium feature --- litellm/proxy/fine_tuning_endpoints/endpoints.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 8c5ceef0183..cda226b5aa7 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -112,12 +112,17 @@ async def create_fine_tuning_job( add_litellm_data_to_request, general_settings, get_custom_headers, + premium_user, proxy_config, proxy_logging_obj, version, ) try: + if premium_user is not True: + raise ValueError( + f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" + ) # Convert Pydantic model to dict data = fine_tuning_request.model_dump(exclude_none=True) @@ -230,6 +235,7 @@ async def list_fine_tuning_jobs( add_litellm_data_to_request, general_settings, get_custom_headers, + premium_user, proxy_config, proxy_logging_obj, version, @@ -237,6 +243,10 @@ async def list_fine_tuning_jobs( data: dict = {} try: + if premium_user is not True: + raise ValueError( + f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" + ) # Include original request and headers in the data data = await add_litellm_data_to_request( data=data, @@ -337,6 +347,7 @@ async def retrieve_fine_tuning_job( add_litellm_data_to_request, general_settings, get_custom_headers, + premium_user, proxy_config, proxy_logging_obj, version, @@ -344,6 +355,10 @@ async def retrieve_fine_tuning_job( data: dict = {} try: + if premium_user is not True: + raise ValueError( + f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" + ) # Include original request and headers in the data data = await add_litellm_data_to_request( data=data, From cd3a01c03fbf009ddb99a1bf67ccd264bb3ce7c2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Wed, 31 Jul 2024 16:46:58 -0700 Subject: [PATCH 31/31] fix fine tuning endpoint postion on swagger --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 983037df370..6a40f0a1097 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -9608,12 +9608,12 @@ def cleanup_router_config_variables(): app.include_router(router) +app.include_router(fine_tuning_router) app.include_router(health_router) app.include_router(key_management_router) app.include_router(internal_user_router) app.include_router(team_router) app.include_router(spend_management_router) -app.include_router(fine_tuning_router) app.include_router(caching_router) app.include_router(analytics_router) app.include_router(debugging_endpoints_router)